class Box(DOMWidget): """Displays multiple widgets in a group.""" _view_name = Unicode('BoxView', sync=True) # Child widgets in the container. # Using a tuple here to force reassignment to update the list. # When a proper notifying-list trait exists, that is what should be used here. children = Tuple(sync=True, allow_none=False) _overflow_values = ['visible', 'hidden', 'scroll', 'auto', 'initial', 'inherit', ''] overflow_x = CaselessStrEnum( values=_overflow_values, default_value='', allow_none=False, sync=True, help="""Specifies what happens to content that is too large for the rendered region.""") overflow_y = CaselessStrEnum( values=_overflow_values, default_value='', allow_none=False, sync=True, help="""Specifies what happens to content that is too large for the rendered region.""") box_style = CaselessStrEnum( values=['success', 'info', 'warning', 'danger', ''], default_value='', allow_none=True, sync=True, help="""Use a predefined styling for the box.""") def __init__(self, children = (), **kwargs): kwargs['children'] = children super(Box, self).__init__(**kwargs) self.on_displayed(Box._fire_children_displayed) def _fire_children_displayed(self): for child in self.children: child._handle_displayed()
class DOMWidget(Widget): visible = Bool(True, allow_none=True, help="Whether the widget is visible. False collapses the empty space, while None preserves the empty space.", sync=True) _css = Tuple(sync=True, help="CSS property list: (selector, key, value)") _dom_classes = Tuple(sync=True, help="DOM classes applied to widget.$el.") width = CUnicode(sync=True) height = CUnicode(sync=True) # A default padding of 2.5 px makes the widgets look nice when displayed inline. padding = CUnicode("2.5px", sync=True) margin = CUnicode(sync=True) color = Unicode(sync=True) background_color = Unicode(sync=True) border_color = Unicode(sync=True) border_width = CUnicode(sync=True) border_radius = CUnicode(sync=True) border_style = CaselessStrEnum(values=[ # http://www.w3schools.com/cssref/pr_border-style.asp 'none', 'hidden', 'dotted', 'dashed', 'solid', 'double', 'groove', 'ridge', 'inset', 'outset', 'initial', 'inherit', ''], default_value='', sync=True) font_style = CaselessStrEnum(values=[ # http://www.w3schools.com/cssref/pr_font_font-style.asp 'normal', 'italic', 'oblique', 'initial', 'inherit', ''], default_value='', sync=True) font_weight = CaselessStrEnum(values=[ # http://www.w3schools.com/cssref/pr_font_weight.asp 'normal', 'bold', 'bolder', 'lighter', 'initial', 'inherit', ''] + [str(100 * (i+1)) for i in range(9)], default_value='', sync=True) font_size = CUnicode(sync=True) font_family = Unicode(sync=True) def __init__(self, *pargs, **kwargs): super(DOMWidget, self).__init__(*pargs, **kwargs) def _validate_border(name, old, new): if new is not None and new != '': if name != 'border_width' and not self.border_width: self.border_width = 1 if name != 'border_style' and self.border_style == '': self.border_style = 'solid' self.on_trait_change(_validate_border, ['border_width', 'border_style', 'border_color'])
class FloatProgress(_BoundedFloat): """ Displays a progress bar. Parameters ----------- value : float position within the range of the progress bar min : float minimal position of the slider max : float maximal position of the slider step : float step of the progress bar description : str name of the progress bar bar_style: {'success', 'info', 'warning', 'danger', ''}, optional color of the progress bar, default is '' (blue) colors are: 'success'-green, 'info'-light blue, 'warning'-orange, 'danger'-red """ _view_name = Unicode('ProgressView', sync=True) bar_style = CaselessStrEnum( values=['success', 'info', 'warning', 'danger', ''], default_value='', allow_none=True, sync=True, help="""Use a predefined styling for the progess bar.""")
class ToggleButton(_Bool): """Displays a boolean `value` in the form of a toggle button. Parameters ---------- value : {True,False} value of the toggle button: True-pressed, False-unpressed description : str description displayed next to the button tooltip: str tooltip caption of the toggle button icon: str font-awesome icon name """ _view_name = Unicode('ToggleButtonView', sync=True) tooltip = Unicode(help="Tooltip caption of the toggle button.", sync=True) icon = Unicode('', help="Font-awesome icon.", sync=True) button_style = CaselessStrEnum( values=['primary', 'success', 'info', 'warning', 'danger', ''], default_value='', allow_none=True, sync=True, help="""Use a predefined styling for the button.""")
class InlineBackend(InlineBackendConfig): """An object to store configuration of the inline backend.""" def _config_changed(self, name, old, new): # warn on change of renamed config section if new.InlineBackendConfig != old.InlineBackendConfig: warn("InlineBackendConfig has been renamed to InlineBackend") super(InlineBackend, self)._config_changed(name, old, new) # The typical default figure size is too large for inline use, # so we shrink the figure size to 6x4, and tweak fonts to # make that fit. rc = Dict( { 'figure.figsize': (6.0, 4.0), # play nicely with white background in the Qt and notebook frontend 'figure.facecolor': 'white', 'figure.edgecolor': 'white', # 12pt labels get cutoff on 6x4 logplots, so use 10pt. 'font.size': 10, # 72 dpi matches SVG/qtconsole # this only affects PNG export, as SVG has no dpi setting 'savefig.dpi': 72, # 10pt still needs a little more room on the xlabel: 'figure.subplot.bottom': .125 }, config=True, help="""Subset of matplotlib rcParams that should be different for the inline backend.""") figure_format = CaselessStrEnum( ['svg', 'png', 'retina'], default_value='png', config=True, help="The image format for figures with the inline backend.") def _figure_format_changed(self, name, old, new): if self.shell is None: return else: select_figure_format(self.shell, new) close_figures = Bool(True, config=True, help="""Close all figures at the end of each cell. When True, ensures that each cell starts with no active figures, but it also means that one must keep track of references in order to edit or redraw figures in subsequent cells. This mode is ideal for the notebook, where residual plots from other cells might be surprising. When False, one must call figure() to create new figures. This means that gcf() and getfigs() can reference figures created in other cells, and the active figure can continue to be edited with pylab/pyplot methods that reference the current active figure. This mode facilitates iterative editing of figures, and behaves most consistently with other matplotlib backends, but figure barriers between cells must be explicit. """) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
class FloatRangeSlider(_BoundedFloatRange): """ Slider/trackbar for displaying a floating value range (within the specified range of values). Parameters ---------- value : float tuple range of the slider displayed min : float minimal position of the slider max : float maximal position of the slider step : float step of the trackbar description : str name of the slider orientation : {'vertical', 'horizontal}, optional default is horizontal readout : {True, False}, optional default is True, display the current value of the slider next to it slider_color : str Unicode color code (eg. '#C13535'), optional color of the slider color : str Unicode color code (eg. '#C13535'), optional color of the value displayed (if readout == True) """ _view_name = Unicode('FloatSliderView', sync=True) orientation = CaselessStrEnum(values=['horizontal', 'vertical'], default_value='horizontal', help="Vertical or horizontal.", sync=True) _range = Bool(True, help="Display a range selector", sync=True) readout = Bool(True, help="Display the current value of the slider next to it.", sync=True) slider_color = Unicode(sync=True)
class InlineBackendConfig(SingletonConfigurable): """An object to store configuration of the inline backend.""" # The typical default figure size is too large for inline use, # so we shrink the figure size to 6x4, and tweak fonts to # make that fit. This is configurable via Global.pylab_inline_rc, # or rather it will be once the zmq kernel is hooked up to # the config system. rc = Dict( { 'figure.figsize': (6.0, 4.0), # 12pt labels get cutoff on 6x4 logplots, so use 10pt. 'font.size': 10, # 10pt still needs a little more room on the xlabel: 'figure.subplot.bottom': .125 }, config=True, help="""Subset of matplotlib rcParams that should be different for the inline backend.""") figure_format = CaselessStrEnum( ['svg', 'png'], default_value='png', config=True, help="The image format for figures with the inline backend.") def _figure_format_changed(self, name, old, new): if self.shell is None: return else: select_figure_format(self.shell, new) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
class SelectizeWidget(Select): _view_name = Unicode('SelectizeView', sync=True) theme = CaselessStrEnum( values=['default', 'legacy', 'bootstrap2', 'bootstrap3'], default_value='default', sync=True, help="""Use a them styling for the SelectizeWidget.""") disabled = Bool(False, help="Enable or disable user changes", sync=True) description = Unicode( help="Description of the value this widget represents", sync=True) def __init__(self, *args, **kwargs): Select.__init__(self, *args, **kwargs) self.value = ','.join(self.values) def _value_name_changed(self, name, old, new): """Called when the value name has been changed (typically by the frontend).""" if self.value_lock.acquire(False): try: self.value_name = self.value = new finally: self.value_lock.release() def _value_changed(self, name, old, new): """Called when value has been changed""" if self.value_lock.acquire(False): try: self.value_name = self.value = new finally: self.value_lock.release()
class IntProgress(_BoundedInt): """Progress bar that represents a int bounded by a minimum and maximum value.""" _view_name = Unicode('ProgressView', sync=True) bar_style = CaselessStrEnum( values=['success', 'info', 'warning', 'danger', ''], default_value='', allow_none=True, sync=True, help="""Use a predefined styling for the progess bar.""")
class IntRangeSlider(_BoundedIntRange): _view_name = Unicode('IntSliderView', sync=True) orientation = CaselessStrEnum(values=['horizontal', 'vertical'], default_value='horizontal', allow_none=False, help="Vertical or horizontal.", sync=True) _range = Bool(True, help="Display a range selector", sync=True) readout = Bool(True, help="Display the current value of the slider next to it.", sync=True) slider_color = Unicode(sync=True)
class FlexBox(Box): """Displays multiple widgets using the flexible box model.""" _view_name = Unicode('FlexBoxView', sync=True) orientation = CaselessStrEnum(values=['vertical', 'horizontal'], default_value='vertical', sync=True) flex = Int(0, sync=True, help="""Specify the flexible-ness of the model.""") def _flex_changed(self, name, old, new): new = min(max(0, new), 2) if self.flex != new: self.flex = new _locations = ['start', 'center', 'end', 'baseline', 'stretch'] pack = CaselessStrEnum( values=_locations, default_value='start', allow_none=False, sync=True) align = CaselessStrEnum( values=_locations, default_value='start', allow_none=False, sync=True)
class Dropdown(_Selection): """Allows you to select a single item from a dropdown.""" _view_name = Unicode('DropdownView', sync=True) button_style = CaselessStrEnum( values=['primary', 'success', 'info', 'warning', 'danger', ''], default_value='', allow_none=True, sync=True, help="""Use a predefined styling for the buttons.""")
class IntSlider(_BoundedInt): """Slider widget that represents a int bounded by a minimum and maximum value.""" _view_name = Unicode('IntSliderView', sync=True) orientation = CaselessStrEnum(values=['horizontal', 'vertical'], default_value='horizontal', help="Vertical or horizontal.", sync=True) _range = Bool(False, help="Display a range selector", sync=True) readout = Bool(True, help="Display the current value of the slider next to it.", sync=True) slider_color = Color(None, allow_none=True, sync=True)
class ToggleButtons(_Selection): """Group of toggle buttons that represent an enumeration. Only one toggle button can be toggled at any point in time.""" _view_name = Unicode('ToggleButtonsView', sync=True) button_style = CaselessStrEnum( values=['primary', 'success', 'info', 'warning', 'danger', ''], default_value='', allow_none=True, sync=True, help="""Use a predefined styling for the buttons.""")
class Button(DOMWidget): """Button widget. This widget has an `on_click` method that allows you to listen for the user clicking on the button. The click event itself is stateless. Parameters ---------- description : str description displayed next to the button tooltip: str tooltip caption of the toggle button icon: str font-awesome icon name """ _view_name = Unicode('ButtonView', sync=True) # Keys description = Unicode('', help="Button label.", sync=True) tooltip = Unicode(help="Tooltip caption of the button.", sync=True) disabled = Bool(False, help="Enable or disable user changes.", sync=True) icon = Unicode('', help="Font-awesome icon.", sync=True) button_style = CaselessStrEnum( values=['primary', 'success', 'info', 'warning', 'danger', ''], default_value='', allow_none=True, sync=True, help="""Use a predefined styling for the button.""") def __init__(self, **kwargs): """Constructor""" super(Button, self).__init__(**kwargs) self._click_handlers = CallbackDispatcher() self.on_msg(self._handle_button_msg) def on_click(self, callback, remove=False): """Register a callback to execute when the button is clicked. The callback will be called with one argument, the clicked button widget instance. Parameters ---------- remove : bool (optional) Set to true to remove the callback from the list of callbacks.""" self._click_handlers.register_callback(callback, remove=remove) def _handle_button_msg(self, _, content): """Handle a msg from the front-end. Parameters ---------- content: dict Content of the msg.""" if content.get('event', '') == 'click': self._click_handlers(self)
class FloatProgress(_BoundedFloat): _view_name = Unicode('ProgressView', sync=True) bar_style = CaselessStrEnum( values=['success', 'info', 'warning', 'danger', ''], default_value='', allow_none=True, sync=True, help="""Use a predefined styling for the progess bar.""")
class ToggleButton(_Bool): """Displays a boolean `value`.""" _view_name = Unicode('ToggleButtonView', sync=True) button_style = CaselessStrEnum( values=['primary', 'success', 'info', 'warning', 'danger', ''], default_value='', allow_none=True, sync=True, help="""Use a predefined styling for the button.""")
class IPKernelApp(KernelApp, InteractiveShellApp): name = 'ipkernel' aliases = Dict(aliases) flags = Dict(flags) classes = [Kernel, ZMQInteractiveShell, ProfileDir, Session] # configurables pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'inline', 'auto'], config=True, help="""Pre-load matplotlib and numpy for interactive use, selecting a particular matplotlib backend and loop integration. """ ) @catch_config_error def initialize(self, argv=None): super(IPKernelApp, self).initialize(argv) self.init_shell() self.init_extensions() self.init_code() def init_kernel(self): kernel_factory = Kernel if self.pylab: gui, backend = pylabtools.find_gui_and_backend(self.pylab) kernel = kernel_factory(config=self.config, session=self.session, shell_socket=self.shell_socket, iopub_socket=self.iopub_socket, stdin_socket=self.stdin_socket, log=self.log, ) self.kernel = kernel kernel.record_ports(self.ports) if self.pylab: kernel.shell.enable_pylab(gui, import_all=self.pylab_import_all) def init_shell(self): self.shell = self.kernel.shell
class NbconvertApp(Application): fmt = CaselessStrEnum(converters.keys(), default_value='rst', config=True, help="Supported conversion format") exclude = List([], config=True, help='list of cells to exclude while converting') aliases = { 'format': 'NbconvertApp.fmt', 'exclude': 'NbconvertApp.exclude', 'highlight': 'Converter.highlight_source', 'preamble': 'Converter.preamble', } def __init__(self, **kwargs): super(NbconvertApp, self).__init__(**kwargs) # ensure those are registerd self.classes.insert(0, Converter) self.classes.insert(0, ConverterRST) self.classes.insert(0, ConverterMarkdown) self.classes.insert(0, ConverterBloggerHTML) self.classes.insert(0, ConverterLaTeX) self.classes.insert(0, ConverterPy) def initialize(self, argv=None): self.parse_command_line(argv) cl_config = self.config self.update_config(cl_config) def run(self): """Convert a notebook in one step""" ConverterClass = converters[self.fmt] infile = (self.extra_args or [None])[0] converter = ConverterClass(infile=infile, config=self.config) converter.render()
class IPKernelApp(BaseIPythonApplication, InteractiveShellApp): name = 'ipkernel' aliases = Dict(kernel_aliases) flags = Dict(kernel_flags) classes = [Kernel, ZMQInteractiveShell, ProfileDir, Session] # the kernel class, as an importstring kernel_class = DottedObjectName('IPython.kernel.zmq.ipkernel.Kernel') kernel = Any() poller = Any( ) # don't restrict this even though current pollers are all Threads heartbeat = Instance(Heartbeat) session = Instance('IPython.kernel.zmq.session.Session') ports = Dict() # inherit config file name from parent: parent_appname = Unicode(config=True) def _parent_appname_changed(self, name, old, new): if self.config_file_specified: # it was manually specified, ignore return self.config_file_name = new.replace('-', '_') + u'_config.py' # don't let this count as specifying the config file self.config_file_specified = False # connection info: transport = CaselessStrEnum(['tcp', 'ipc'], default_value='tcp', config=True) ip = Unicode( config=True, help="Set the IP or interface on which the kernel will listen.") def _ip_default(self): if self.transport == 'ipc': if self.connection_file: return os.path.splitext(self.abs_connection_file)[0] + '-ipc' else: return 'kernel-ipc' else: return LOCALHOST hb_port = Integer(0, config=True, help="set the heartbeat port [default: random]") shell_port = Integer(0, config=True, help="set the shell (ROUTER) port [default: random]") iopub_port = Integer(0, config=True, help="set the iopub (PUB) port [default: random]") stdin_port = Integer(0, config=True, help="set the stdin (DEALER) port [default: random]") connection_file = Unicode( '', config=True, help= """JSON file in which to store connection info [default: kernel-<pid>.json] This file will contain the IP, ports, and authentication key needed to connect clients to this kernel. By default, this file will be created in the security dir of the current profile, but can be specified by absolute path. """) @property def abs_connection_file(self): if os.path.basename(self.connection_file) == self.connection_file: return os.path.join(self.profile_dir.security_dir, self.connection_file) else: return self.connection_file # streams, etc. no_stdout = Bool(False, config=True, help="redirect stdout to the null device") no_stderr = Bool(False, config=True, help="redirect stderr to the null device") outstream_class = DottedObjectName( 'IPython.kernel.zmq.iostream.OutStream', config=True, help="The importstring for the OutStream factory") displayhook_class = DottedObjectName( 'IPython.kernel.zmq.displayhook.ZMQDisplayHook', config=True, help="The importstring for the DisplayHook factory") # polling parent = Integer( 0, config=True, help="""kill this process if its parent dies. On Windows, the argument specifies the HANDLE of the parent process, otherwise it is simply boolean. """) interrupt = Integer(0, config=True, help="""ONLY USED ON WINDOWS Interrupt this process when the parent is signaled. """) def init_crash_handler(self): # Install minimal exception handling sys.excepthook = FormattedTB(mode='Verbose', color_scheme='NoColor', ostream=sys.__stdout__) def init_poller(self): if sys.platform == 'win32': if self.interrupt or self.parent: self.poller = ParentPollerWindows(self.interrupt, self.parent) elif self.parent: self.poller = ParentPollerUnix() def _bind_socket(self, s, port): iface = '%s://%s' % (self.transport, self.ip) if self.transport == 'tcp': if port <= 0: port = s.bind_to_random_port(iface) else: s.bind("tcp://%s:%i" % (self.ip, port)) elif self.transport == 'ipc': if port <= 0: port = 1 path = "%s-%i" % (self.ip, port) while os.path.exists(path): port = port + 1 path = "%s-%i" % (self.ip, port) else: path = "%s-%i" % (self.ip, port) s.bind("ipc://%s" % path) return port def load_connection_file(self): """load ip/port/hmac config from JSON connection file""" try: fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir]) except IOError: self.log.debug("Connection file not found: %s", self.connection_file) # This means I own it, so I will clean it up: atexit.register(self.cleanup_connection_file) return self.log.debug(u"Loading connection file %s", fname) with open(fname) as f: s = f.read() cfg = json.loads(s) self.transport = cfg.get('transport', self.transport) if self.ip == self._ip_default() and 'ip' in cfg: # not overridden by config or cl_args self.ip = cfg['ip'] for channel in ('hb', 'shell', 'iopub', 'stdin'): name = channel + '_port' if getattr(self, name) == 0 and name in cfg: # not overridden by config or cl_args setattr(self, name, cfg[name]) if 'key' in cfg: self.config.Session.key = str_to_bytes(cfg['key']) def write_connection_file(self): """write connection info to JSON file""" cf = self.abs_connection_file self.log.debug("Writing connection file: %s", cf) write_connection_file(cf, ip=self.ip, key=self.session.key, transport=self.transport, shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port, iopub_port=self.iopub_port) def cleanup_connection_file(self): cf = self.abs_connection_file self.log.debug("Cleaning up connection file: %s", cf) try: os.remove(cf) except (IOError, OSError): pass self.cleanup_ipc_files() def cleanup_ipc_files(self): """cleanup ipc files if we wrote them""" if self.transport != 'ipc': return for port in (self.shell_port, self.iopub_port, self.stdin_port, self.hb_port): ipcfile = "%s-%i" % (self.ip, port) try: os.remove(ipcfile) except (IOError, OSError): pass def init_connection_file(self): if not self.connection_file: self.connection_file = "kernel-%s.json" % os.getpid() try: self.load_connection_file() except Exception: self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) self.exit(1) def init_sockets(self): # Create a context, a session, and the kernel sockets. self.log.info("Starting the kernel at pid: %i", os.getpid()) context = zmq.Context.instance() # Uncomment this to try closing the context. # atexit.register(context.term) self.shell_socket = context.socket(zmq.ROUTER) self.shell_port = self._bind_socket(self.shell_socket, self.shell_port) self.log.debug("shell ROUTER Channel on port: %i" % self.shell_port) self.iopub_socket = context.socket(zmq.PUB) self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port) self.log.debug("iopub PUB Channel on port: %i" % self.iopub_port) self.stdin_socket = context.socket(zmq.ROUTER) self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port) self.log.debug("stdin ROUTER Channel on port: %i" % self.stdin_port) def init_heartbeat(self): """start the heart beating""" # heartbeat doesn't share context, because it mustn't be blocked # by the GIL, which is accessed by libzmq when freeing zero-copy messages hb_ctx = zmq.Context() self.heartbeat = Heartbeat(hb_ctx, (self.transport, self.ip, self.hb_port)) self.hb_port = self.heartbeat.port self.log.debug("Heartbeat REP Channel on port: %i" % self.hb_port) self.heartbeat.start() # Helper to make it easier to connect to an existing kernel. # set log-level to critical, to make sure it is output self.log.critical("To connect another client to this kernel, use:") def log_connection_info(self): """display connection info, and store ports""" basename = os.path.basename(self.connection_file) if basename == self.connection_file or \ os.path.dirname(self.connection_file) == self.profile_dir.security_dir: # use shortname tail = basename if self.profile != 'default': tail += " --profile %s" % self.profile else: tail = self.connection_file self.log.critical("--existing %s", tail) self.ports = dict(shell=self.shell_port, iopub=self.iopub_port, stdin=self.stdin_port, hb=self.hb_port) def init_session(self): """create our session object""" default_secure(self.config) self.session = Session(config=self.config, username=u'kernel') def init_blackhole(self): """redirects stdout/stderr to devnull if necessary""" if self.no_stdout or self.no_stderr: blackhole = open(os.devnull, 'w') if self.no_stdout: sys.stdout = sys.__stdout__ = blackhole if self.no_stderr: sys.stderr = sys.__stderr__ = blackhole def init_io(self): """Redirect input streams and set a display hook.""" if self.outstream_class: outstream_factory = import_item(str(self.outstream_class)) sys.stdout = outstream_factory(self.session, self.iopub_socket, u'stdout') sys.stderr = outstream_factory(self.session, self.iopub_socket, u'stderr') if self.displayhook_class: displayhook_factory = import_item(str(self.displayhook_class)) sys.displayhook = displayhook_factory(self.session, self.iopub_socket) def init_signal(self): signal.signal(signal.SIGINT, signal.SIG_IGN) def init_kernel(self): """Create the Kernel object itself""" shell_stream = ZMQStream(self.shell_socket) kernel = Kernel( config=self.config, session=self.session, shell_streams=[shell_stream], iopub_socket=self.iopub_socket, stdin_socket=self.stdin_socket, log=self.log, profile_dir=self.profile_dir, ) kernel.record_ports(self.ports) self.kernel = kernel def init_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" # Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab` # to ensure that any exception is printed straight to stderr. # Normally _showtraceback associates the reply with an execution, # which means frontends will never draw it, as this exception # is not associated with any execute request. shell = self.shell _showtraceback = shell._showtraceback try: # replace pyerr-sending traceback with stderr def print_tb(etype, evalue, stb): print("GUI event loop or pylab initialization failed", file=io.stderr) print(shell.InteractiveTB.stb2text(stb), file=io.stderr) shell._showtraceback = print_tb InteractiveShellApp.init_gui_pylab(self) finally: shell._showtraceback = _showtraceback def init_shell(self): self.shell = self.kernel.shell self.shell.configurables.append(self) @catch_config_error def initialize(self, argv=None): super(IPKernelApp, self).initialize(argv) self.init_blackhole() self.init_connection_file() self.init_session() self.init_poller() self.init_sockets() self.init_heartbeat() # writing/displaying connection info must be *after* init_sockets/heartbeat self.log_connection_info() self.write_connection_file() self.init_io() self.init_signal() self.init_kernel() # shell init steps self.init_path() self.init_shell() self.init_gui_pylab() self.init_extensions() self.init_code() # flush stdout/stderr, so that anything written to these streams during # initialization do not get associated with the first execution request sys.stdout.flush() sys.stderr.flush() def start(self): if self.poller is not None: self.poller.start() self.kernel.start() try: ioloop.IOLoop.instance().start() except KeyboardInterrupt: pass
class IPKernelApp(KernelApp, InteractiveShellApp): name = 'ipkernel' aliases = Dict(aliases) flags = Dict(flags) classes = [Kernel, ZMQInteractiveShell, ProfileDir, Session] # configurables pylab = CaselessStrEnum( ['tk', 'qt', 'wx', 'gtk', 'osx', 'inline', 'auto'], config=True, help="""Pre-load matplotlib and numpy for interactive use, selecting a particular matplotlib backend and loop integration. """) @catch_config_error def initialize(self, argv=None): super(IPKernelApp, self).initialize(argv) self.init_shell() self.init_extensions() self.init_code() def init_kernel(self): kernel = Kernel( config=self.config, session=self.session, shell_socket=self.shell_socket, iopub_socket=self.iopub_socket, stdin_socket=self.stdin_socket, log=self.log, profile_dir=self.profile_dir, ) self.kernel = kernel kernel.record_ports(self.ports) shell = kernel.shell if self.pylab: try: gui, backend = pylabtools.find_gui_and_backend(self.pylab) shell.enable_pylab(gui, import_all=self.pylab_import_all) except Exception: self.log.error("Pylab initialization failed", exc_info=True) # print exception straight to stdout, because normally # _showtraceback associates the reply with an execution, # which means frontends will never draw it, as this exception # is not associated with any execute request. # replace pyerr-sending traceback with stdout _showtraceback = shell._showtraceback def print_tb(etype, evalue, stb): print( "Error initializing pylab, pylab mode will not " "be active", file=io.stderr) print(shell.InteractiveTB.stb2text(stb), file=io.stdout) shell._showtraceback = print_tb # send the traceback over stdout shell.showtraceback(tb_offset=0) # restore proper _showtraceback method shell._showtraceback = _showtraceback def init_shell(self): self.shell = self.kernel.shell self.shell.configurables.append(self)
class KernelManager(HasTraits): """ Manages a kernel for a frontend. The SUB channel is for the frontend to receive messages published by the kernel. The REQ channel is for the frontend to make requests of the kernel. The REP channel is for the kernel to request stdin (raw_input) from the frontend. """ # config object for passing to child configurables config = Instance(Config) # The PyZMQ Context to use for communication with the kernel. context = Instance(zmq.Context) def _context_default(self): return zmq.Context.instance() # The Session to use for communication with the kernel. session = Instance(Session) # The kernel process with which the KernelManager is communicating. kernel = Instance(Popen) # The addresses for the communication channels. connection_file = Unicode('') transport = CaselessStrEnum(['tcp', 'ipc'], default_value='tcp') ip = Unicode(LOCALHOST) def _ip_changed(self, name, old, new): if new == '*': self.ip = '0.0.0.0' shell_port = Integer(0) iopub_port = Integer(0) stdin_port = Integer(0) hb_port = Integer(0) # The classes to use for the various channels. shell_channel_class = Type(ShellSocketChannel) sub_channel_class = Type(SubSocketChannel) stdin_channel_class = Type(StdInSocketChannel) hb_channel_class = Type(HBSocketChannel) # Protected traits. _launch_args = Any _shell_channel = Any _sub_channel = Any _stdin_channel = Any _hb_channel = Any _connection_file_written = Bool(False) def __init__(self, **kwargs): super(KernelManager, self).__init__(**kwargs) if self.session is None: self.session = Session(config=self.config) def __del__(self): self.cleanup_connection_file() #-------------------------------------------------------------------------- # Channel management methods: #-------------------------------------------------------------------------- def start_channels(self, shell=True, sub=True, stdin=True, hb=True): """Starts the channels for this kernel. This will create the channels if they do not exist and then start them. If port numbers of 0 are being used (random ports) then you must first call :method:`start_kernel`. If the channels have been stopped and you call this, :class:`RuntimeError` will be raised. """ if shell: self.shell_channel.start() if sub: self.sub_channel.start() if stdin: self.stdin_channel.start() self.shell_channel.allow_stdin = True else: self.shell_channel.allow_stdin = False if hb: self.hb_channel.start() def stop_channels(self): """Stops all the running channels for this kernel. """ if self.shell_channel.is_alive(): self.shell_channel.stop() if self.sub_channel.is_alive(): self.sub_channel.stop() if self.stdin_channel.is_alive(): self.stdin_channel.stop() if self.hb_channel.is_alive(): self.hb_channel.stop() @property def channels_running(self): """Are any of the channels created and running?""" return (self.shell_channel.is_alive() or self.sub_channel.is_alive() or self.stdin_channel.is_alive() or self.hb_channel.is_alive()) #-------------------------------------------------------------------------- # Kernel process management methods: #-------------------------------------------------------------------------- def cleanup_connection_file(self): """cleanup connection file *if we wrote it* Will not raise if the connection file was already removed somehow. """ if self._connection_file_written: # cleanup connection files on full shutdown of kernel we started self._connection_file_written = False try: os.remove(self.connection_file) except (IOError, OSError): pass self._cleanup_ipc_files() def _cleanup_ipc_files(self): """cleanup ipc files if we wrote them""" if self.transport != 'ipc': return for port in (self.shell_port, self.iopub_port, self.stdin_port, self.hb_port): ipcfile = "%s-%i" % (self.ip, port) try: os.remove(ipcfile) except (IOError, OSError): pass def load_connection_file(self): """load connection info from JSON dict in self.connection_file""" with open(self.connection_file) as f: cfg = json.loads(f.read()) from pprint import pprint pprint(cfg) self.transport = cfg.get('transport', 'tcp') self.ip = cfg['ip'] self.shell_port = cfg['shell_port'] self.stdin_port = cfg['stdin_port'] self.iopub_port = cfg['iopub_port'] self.hb_port = cfg['hb_port'] self.session.key = str_to_bytes(cfg['key']) def write_connection_file(self): """write connection info to JSON dict in self.connection_file""" if self._connection_file_written: return self.connection_file, cfg = write_connection_file( self.connection_file, transport=self.transport, ip=self.ip, key=self.session.key, stdin_port=self.stdin_port, iopub_port=self.iopub_port, shell_port=self.shell_port, hb_port=self.hb_port) # write_connection_file also sets default ports: self.shell_port = cfg['shell_port'] self.stdin_port = cfg['stdin_port'] self.iopub_port = cfg['iopub_port'] self.hb_port = cfg['hb_port'] self._connection_file_written = True def start_kernel(self, **kw): """Starts a kernel process and configures the manager to use it. If random ports (port=0) are being used, this method must be called before the channels are created. Parameters: ----------- launcher : callable, optional (default None) A custom function for launching the kernel process (generally a wrapper around ``entry_point.base_launch_kernel``). In most cases, it should not be necessary to use this parameter. **kw : optional See respective options for IPython and Python kernels. """ if self.transport == 'tcp' and self.ip not in LOCAL_IPS: raise RuntimeError( "Can only launch a kernel on a local interface. " "Make sure that the '*_address' attributes are " "configured properly. " "Currently valid addresses are: %s" % LOCAL_IPS) # write connection file / get default ports self.write_connection_file() self._launch_args = kw.copy() launch_kernel = kw.pop('launcher', None) if launch_kernel is None: # from ipkernel import launch_kernel from ext_entry_point import launch_kernel self.kernel = launch_kernel(fname=self.connection_file, **kw) def shutdown_kernel(self, restart=False): """ Attempts to the stop the kernel process cleanly. If the kernel cannot be stopped and the kernel is local, it is killed. """ # FIXME: Shutdown does not work on Windows due to ZMQ errors! if sys.platform == 'win32': self.kill_kernel() return # Pause the heart beat channel if it exists. if self._hb_channel is not None: self._hb_channel.pause() # Don't send any additional kernel kill messages immediately, to give # the kernel a chance to properly execute shutdown actions. Wait for at # most 1s, checking every 0.1s. self.shell_channel.shutdown(restart=restart) for i in range(10): if self.is_alive: time.sleep(0.1) else: break else: # OK, we've waited long enough. if self.has_kernel: self.kill_kernel() if not restart and self._connection_file_written: # cleanup connection files on full shutdown of kernel we started self._connection_file_written = False try: os.remove(self.connection_file) except IOError: pass def restart_kernel(self, now=False, **kw): """Restarts a kernel with the arguments that were used to launch it. If the old kernel was launched with random ports, the same ports will be used for the new kernel. Parameters ---------- now : bool, optional If True, the kernel is forcefully restarted *immediately*, without having a chance to do any cleanup action. Otherwise the kernel is given 1s to clean up before a forceful restart is issued. In all cases the kernel is restarted, the only difference is whether it is given a chance to perform a clean shutdown or not. **kw : optional Any options specified here will replace those used to launch the kernel. """ if self._launch_args is None: raise RuntimeError("Cannot restart the kernel. " "No previous call to 'start_kernel'.") else: # Stop currently running kernel. if self.has_kernel: if now: self.kill_kernel() else: self.shutdown_kernel(restart=True) # Start new kernel. self._launch_args.update(kw) self.start_kernel(**self._launch_args) # FIXME: Messages get dropped in Windows due to probable ZMQ bug # unless there is some delay here. if sys.platform == 'win32': time.sleep(0.2) @property def has_kernel(self): """Returns whether a kernel process has been specified for the kernel manager. """ return self.kernel is not None def kill_kernel(self): """ Kill the running kernel. This method blocks until the kernel process has terminated. """ if self.has_kernel: # Pause the heart beat channel if it exists. if self._hb_channel is not None: self._hb_channel.pause() # Signal the kernel to terminate (sends SIGKILL on Unix and calls # TerminateProcess() on Win32). try: self.kernel.kill() except OSError as e: # In Windows, we will get an Access Denied error if the process # has already terminated. Ignore it. if sys.platform == 'win32': if e.winerror != 5: raise # On Unix, we may get an ESRCH error if the process has already # terminated. Ignore it. else: from errno import ESRCH if e.errno != ESRCH: raise # Block until the kernel terminates. self.kernel.wait() self.kernel = None else: raise RuntimeError("Cannot kill kernel. No kernel is running!") def interrupt_kernel(self): """ Interrupts the kernel. Unlike ``signal_kernel``, this operation is well supported on all platforms. """ if self.has_kernel: if sys.platform == 'win32': from parentpoller import ParentPollerWindows as Poller Poller.send_interrupt(self.kernel.win32_interrupt_event) else: self.kernel.send_signal(signal.SIGINT) else: raise RuntimeError( "Cannot interrupt kernel. No kernel is running!") def signal_kernel(self, signum): """ Sends a signal to the kernel. Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. """ if self.has_kernel: self.kernel.send_signal(signum) else: raise RuntimeError("Cannot signal kernel. No kernel is running!") @property def is_alive(self): """Is the kernel process still running?""" if self.has_kernel: if self.kernel.poll() is None: return True else: return False elif self._hb_channel is not None: # We didn't start the kernel with this KernelManager so we # use the heartbeat. return self._hb_channel.is_beating() else: # no heartbeat and not local, we can't tell if it's running, # so naively return True return True #-------------------------------------------------------------------------- # Channels used for communication with the kernel: #-------------------------------------------------------------------------- def _make_url(self, port): """make a zmq url with a port""" if self.transport == 'tcp': return "tcp://%s:%i" % (self.ip, port) else: return "%s://%s-%s" % (self.transport, self.ip, port) @property def shell_channel(self): """Get the REQ socket channel object to make requests of the kernel.""" if self._shell_channel is None: self._shell_channel = self.shell_channel_class( self.context, self.session, self._make_url(self.shell_port), ) return self._shell_channel @property def sub_channel(self): """Get the SUB socket channel object.""" if self._sub_channel is None: self._sub_channel = self.sub_channel_class( self.context, self.session, self._make_url(self.iopub_port), ) return self._sub_channel @property def stdin_channel(self): """Get the REP socket channel object to handle stdin (raw_input).""" if self._stdin_channel is None: self._stdin_channel = self.stdin_channel_class( self.context, self.session, self._make_url(self.stdin_port), ) return self._stdin_channel @property def hb_channel(self): """Get the heartbeat socket channel object to check that the kernel is alive.""" if self._hb_channel is None: self._hb_channel = self.hb_channel_class( self.context, self.session, self._make_url(self.hb_port), ) return self._hb_channel
class ExecutionContext(LoggingConfigurable): # These first are valid for the time of the existance of this contex output = Instance(klass=TemporaryOutputDocument, allow_none=True, config=False, help="current output document") chunk_number = Integer(0, config=False, allow_none=False, help="current chunk number") def _chunk_number_changed(self, name, old, new): if old != new: self.chunk_label = None enabled_documents = List([], config=False, help="Names for enabled documents.") # the following are valid for the time of one code execution chunk_label = Unicode(None, config=False, allow_none=True, help="current chunk label") chunk_plot_number = Integer(0, config=False, allow_none=False, help="current plot number in this chunk") def _chunk_label_changed(self, name, old, new): if old != new: self.chunk_plot_number = 0 echo = Bool(True, config=False, help="If False, knitpy will not display the code in the code " "chunk above it's results in the final document.") results = CaselessStrEnum( default_value="markup", values=["markup", "hide", "hold", "asis"], allow_none=False, config=False, help="If 'hide', knitpy will not display the code’s results in the " "final document. If 'hold', knitpy will delay displaying all " "output pieces until the end of the chunk. If 'asis', " "knitpy will pass through results without reformatting them " "(useful if results return raw HTML, etc.)") include = Bool(True, config=False, help="If False, knitpy will will run the chunk but not " "include the chunk in the final document.") comment = Unicode( default_value="##", config=False, allow_none=True, help= "Prefix which is added to all (text) output; None or empty string will " "result in no prefix") mode = CaselessStrEnum(default_value=None, values=["inline", "block"], allow_none=True, config=False, help="current mode: inline or " "block") engine = Instance(klass=BaseKnitpyEngine, allow_none=True, config=False, help="current engine") def __init__(self, output, **kwargs): super(ExecutionContext, self).__init__(**kwargs) self.output = output output.context = self def execution_started(self): self.chunk_number += 1 def execution_finished(self): self.output.flush() reset_needed = [ "engine", "mode" "chunk_label", "comment", "include", "echo", "include", "results" ] for name in self.trait_names(): if name in reset_needed: self.traits()[name].set_default_value(self)
class InteractiveShellApp(Configurable): """A Mixin for applications that start InteractiveShell instances. Provides configurables for loading extensions and executing files as part of configuring a Shell environment. The following methods should be called by the :meth:`initialize` method of the subclass: - :meth:`init_path` - :meth:`init_shell` (to be implemented by the subclass) - :meth:`init_gui_pylab` - :meth:`init_extensions` - :meth:`init_code` """ extensions = List( Unicode, config=True, help="A list of dotted module names of IPython extensions to load.") extra_extension = Unicode( '', config=True, help="dotted module name of an IPython extension to load.") def _extra_extension_changed(self, name, old, new): if new: # add to self.extensions self.extensions.append(new) # Extensions that are always loaded (not configurable) default_extensions = List(Unicode, [u'storemagic'], config=False) hide_initial_ns = Bool( True, config=True, help= """Should variables loaded at startup (by startup files, exec_lines, etc.) be hidden from tools like %who?""") exec_files = List(Unicode, config=True, help="""List of files to run at IPython startup.""") exec_PYTHONSTARTUP = Bool( True, config=True, help="""Run the file referenced by the PYTHONSTARTUP environment variable at IPython startup.""") file_to_run = Unicode('', config=True, help="""A file to be run""") exec_lines = List(Unicode, config=True, help="""lines of code to run at IPython startup.""") code_to_run = Unicode('', config=True, help="Execute the given command string.") module_to_run = Unicode('', config=True, help="Run the module as a script.") gui = CaselessStrEnum( gui_keys, config=True, help="Enable GUI event loop integration with any of {0}.".format( gui_keys)) matplotlib = CaselessStrEnum( backend_keys, config=True, help="""Configure matplotlib for interactive use with the default matplotlib backend.""") pylab = CaselessStrEnum( backend_keys, config=True, help="""Pre-load matplotlib and numpy for interactive use, selecting a particular matplotlib backend and loop integration. """) pylab_import_all = Bool( True, config=True, help= """If true, IPython will populate the user namespace with numpy, pylab, etc. and an ``import *`` is done from numpy and pylab, when using pylab mode. When False, pylab mode should not import any names into the user namespace. """) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') user_ns = Instance(dict, args=None, allow_none=True) def _user_ns_changed(self, name, old, new): if self.shell is not None: self.shell.user_ns = new self.shell.init_user_ns() def init_path(self): """Add current working directory, '', to sys.path""" if sys.path[0] != '': sys.path.insert(0, '') def init_shell(self): raise NotImplementedError("Override in subclasses") def init_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" enable = False shell = self.shell if self.pylab: enable = lambda key: shell.enable_pylab( key, import_all=self.pylab_import_all) key = self.pylab elif self.matplotlib: enable = shell.enable_matplotlib key = self.matplotlib elif self.gui: enable = shell.enable_gui key = self.gui if not enable: return try: r = enable(key) except ImportError: self.log.warn( "Eventloop or matplotlib integration failed. Is matplotlib installed?" ) self.shell.showtraceback() return except Exception: self.log.warn("GUI event loop or pylab initialization failed") self.shell.showtraceback() return if isinstance(r, tuple): gui, backend = r[:2] self.log.info( "Enabling GUI event loop integration, " "eventloop=%s, matplotlib=%s", gui, backend) if key == "auto": print("Using matplotlib backend: %s" % backend) else: gui = r self.log.info( "Enabling GUI event loop integration, " "eventloop=%s", gui) def init_extensions(self): """Load all IPython extensions in IPythonApp.extensions. This uses the :meth:`ExtensionManager.load_extensions` to load all the extensions listed in ``self.extensions``. """ try: self.log.debug("Loading IPython extensions...") extensions = self.default_extensions + self.extensions for ext in extensions: try: self.log.info("Loading IPython extension: %s" % ext) self.shell.extension_manager.load_extension(ext) except: self.log.warn("Error in loading extension: %s" % ext + "\nCheck your config files in %s" % self.profile_dir.location) self.shell.showtraceback() except: self.log.warn("Unknown error in loading extensions:") self.shell.showtraceback() def init_code(self): """run the pre-flight code, specified via exec_lines""" self._run_startup_files() self._run_exec_lines() self._run_exec_files() # Hide variables defined here from %who etc. if self.hide_initial_ns: self.shell.user_ns_hidden.update(self.shell.user_ns) # command-line execution (ipython -i script.py, ipython -m module) # should *not* be excluded from %whos self._run_cmd_line_code() self._run_module() # flush output, so itwon't be attached to the first cell sys.stdout.flush() sys.stderr.flush() def _run_exec_lines(self): """Run lines of code in IPythonApp.exec_lines in the user's namespace.""" if not self.exec_lines: return try: self.log.debug("Running code from IPythonApp.exec_lines...") for line in self.exec_lines: try: self.log.info("Running code in user namespace: %s" % line) self.shell.run_cell(line, store_history=False) except: self.log.warn("Error in executing line in user " "namespace: %s" % line) self.shell.showtraceback() except: self.log.warn("Unknown error in handling IPythonApp.exec_lines:") self.shell.showtraceback() def _exec_file(self, fname, shell_futures=False): try: full_filename = filefind(fname, [u'.', self.ipython_dir]) except IOError as e: self.log.warn("File not found: %r" % fname) return # Make sure that the running script gets a proper sys.argv as if it # were run from a system shell. save_argv = sys.argv sys.argv = [full_filename] + self.extra_args[1:] # protect sys.argv from potential unicode strings on Python 2: if not py3compat.PY3: sys.argv = [py3compat.cast_bytes(a) for a in sys.argv] try: if os.path.isfile(full_filename): self.log.info("Running file in user namespace: %s" % full_filename) # Ensure that __file__ is always defined to match Python # behavior. with preserve_keys(self.shell.user_ns, '__file__'): self.shell.user_ns['__file__'] = fname if full_filename.endswith('.ipy'): self.shell.safe_execfile_ipy( full_filename, shell_futures=shell_futures) else: # default to python, even without extension self.shell.safe_execfile(full_filename, self.shell.user_ns, shell_futures=shell_futures) finally: sys.argv = save_argv def _run_startup_files(self): """Run files from profile startup directory""" startup_dir = self.profile_dir.startup_dir startup_files = [] if self.exec_PYTHONSTARTUP and os.environ.get('PYTHONSTARTUP', False) and \ not (self.file_to_run or self.code_to_run or self.module_to_run): python_startup = os.environ['PYTHONSTARTUP'] self.log.debug("Running PYTHONSTARTUP file %s...", python_startup) try: self._exec_file(python_startup) except: self.log.warn( "Unknown error in handling PYTHONSTARTUP file %s:", python_startup) self.shell.showtraceback() finally: # Many PYTHONSTARTUP files set up the readline completions, # but this is often at odds with IPython's own completions. # Do not allow PYTHONSTARTUP to set up readline. if self.shell.has_readline: self.shell.set_readline_completer() startup_files += glob.glob(os.path.join(startup_dir, '*.py')) startup_files += glob.glob(os.path.join(startup_dir, '*.ipy')) if not startup_files: return self.log.debug("Running startup files from %s...", startup_dir) try: for fname in sorted(startup_files): self._exec_file(fname) except: self.log.warn("Unknown error in handling startup files:") self.shell.showtraceback() def _run_exec_files(self): """Run files from IPythonApp.exec_files""" if not self.exec_files: return self.log.debug("Running files in IPythonApp.exec_files...") try: for fname in self.exec_files: self._exec_file(fname) except: self.log.warn("Unknown error in handling IPythonApp.exec_files:") self.shell.showtraceback() def _run_cmd_line_code(self): """Run code or file specified at the command-line""" if self.code_to_run: line = self.code_to_run try: self.log.info("Running code given at command line (c=): %s" % line) self.shell.run_cell(line, store_history=False) except: self.log.warn("Error in executing line in user namespace: %s" % line) self.shell.showtraceback() # Like Python itself, ignore the second if the first of these is present elif self.file_to_run: fname = self.file_to_run try: self._exec_file(fname, shell_futures=True) except: self.log.warn("Error in executing file in user namespace: %s" % fname) self.shell.showtraceback() def _run_module(self): """Run module specified at the command-line.""" if self.module_to_run: # Make sure that the module gets a proper sys.argv as if it were # run using `python -m`. save_argv = sys.argv sys.argv = [sys.executable] + self.extra_args try: self.shell.safe_run_module(self.module_to_run, self.shell.user_ns) finally: sys.argv = save_argv
class ConnectionFileMixin(Configurable): """Mixin for configurable classes that work with connection files""" # The addresses for the communication channels connection_file = Unicode('') _connection_file_written = Bool(False) transport = CaselessStrEnum(['tcp', 'ipc'], default_value='tcp', config=True) ip = Unicode(config=True, help="""Set the kernel\'s IP address [default localhost]. If the IP address is something other than localhost, then Consoles on other machines will be able to connect to the Kernel, so be careful!""") def _ip_default(self): if self.transport == 'ipc': if self.connection_file: return os.path.splitext(self.connection_file)[0] + '-ipc' else: return 'kernel-ipc' else: return localhost() def _ip_changed(self, name, old, new): if new == '*': self.ip = '0.0.0.0' # protected traits shell_port = Integer(0) iopub_port = Integer(0) stdin_port = Integer(0) control_port = Integer(0) hb_port = Integer(0) @property def ports(self): return [getattr(self, name) for name in port_names] #-------------------------------------------------------------------------- # Connection and ipc file management #-------------------------------------------------------------------------- def get_connection_info(self): """return the connection info as a dict""" return dict( transport=self.transport, ip=self.ip, shell_port=self.shell_port, iopub_port=self.iopub_port, stdin_port=self.stdin_port, hb_port=self.hb_port, control_port=self.control_port, signature_scheme=self.session.signature_scheme, key=self.session.key, ) def cleanup_connection_file(self): """Cleanup connection file *if we wrote it* Will not raise if the connection file was already removed somehow. """ if self._connection_file_written: # cleanup connection files on full shutdown of kernel we started self._connection_file_written = False try: os.remove(self.connection_file) except (IOError, OSError, AttributeError): pass def cleanup_ipc_files(self): """Cleanup ipc files if we wrote them.""" if self.transport != 'ipc': return for port in self.ports: ipcfile = "%s-%i" % (self.ip, port) try: os.remove(ipcfile) except (IOError, OSError): pass def write_connection_file(self): """Write connection info to JSON dict in self.connection_file.""" if self._connection_file_written: return self.connection_file, cfg = write_connection_file( self.connection_file, transport=self.transport, ip=self.ip, key=self.session.key, stdin_port=self.stdin_port, iopub_port=self.iopub_port, shell_port=self.shell_port, hb_port=self.hb_port, control_port=self.control_port, signature_scheme=self.session.signature_scheme, ) # write_connection_file also sets default ports: for name in port_names: setattr(self, name, cfg[name]) self._connection_file_written = True def load_connection_file(self): """Load connection info from JSON dict in self.connection_file.""" with open(self.connection_file) as f: cfg = json.loads(f.read()) self.transport = cfg.get('transport', 'tcp') self.ip = cfg['ip'] for name in port_names: setattr(self, name, cfg[name]) if 'key' in cfg: self.session.key = str_to_bytes(cfg['key']) if cfg.get('signature_scheme'): self.session.signature_scheme = cfg['signature_scheme'] #-------------------------------------------------------------------------- # Creating connected sockets #-------------------------------------------------------------------------- def _make_url(self, channel): """Make a ZeroMQ URL for a given channel.""" transport = self.transport ip = self.ip port = getattr(self, '%s_port' % channel) if transport == 'tcp': return "tcp://%s:%i" % (ip, port) else: return "%s://%s-%s" % (transport, ip, port) def _create_connected_socket(self, channel, identity=None): """Create a zmq Socket and connect it to the kernel.""" url = self._make_url(channel) socket_type = channel_socket_types[channel] self.log.info("Connecting to: %s" % url) sock = self.context.socket(socket_type) if identity: sock.identity = identity sock.connect(url) return sock def connect_iopub(self, identity=None): """return zmq Socket connected to the IOPub channel""" sock = self._create_connected_socket('iopub', identity=identity) sock.setsockopt(zmq.SUBSCRIBE, b'') return sock def connect_shell(self, identity=None): """return zmq Socket connected to the Shell channel""" return self._create_connected_socket('shell', identity=identity) def connect_stdin(self, identity=None): """return zmq Socket connected to the StdIn channel""" return self._create_connected_socket('stdin', identity=identity) def connect_hb(self, identity=None): """return zmq Socket connected to the Heartbeat channel""" return self._create_connected_socket('hb', identity=identity) def connect_control(self, identity=None): """return zmq Socket connected to the Heartbeat channel""" return self._create_connected_socket('control', identity=identity)
class NbConvertApp(BaseIPythonApplication): """Application used to convert from notebook file type (``*.ipynb``)""" name = 'ipython-nbconvert' aliases = nbconvert_aliases flags = nbconvert_flags def _log_level_default(self): return logging.INFO def _classes_default(self): classes = [NbConvertBase, ProfileDir] for pkg in (exporters, preprocessors, writers, postprocessors): for name in dir(pkg): cls = getattr(pkg, name) if isinstance(cls, type) and issubclass(cls, Configurable): classes.append(cls) return classes description = Unicode( u"""This application is used to convert notebook files (*.ipynb) to various other formats. WARNING: THE COMMANDLINE INTERFACE MAY CHANGE IN FUTURE RELEASES.""") output_base = Unicode('', config=True, help='''overwrite base name use for output files. can only be used when converting one notebook at a time. ''') use_output_suffix = Bool( True, config=True, help="""Whether to apply a suffix prior to the extension (only relevant when converting to notebook format). The suffix is determined by the exporter, and is usually '.nbconvert'.""") examples = Unicode(u""" The simplest way to use nbconvert is > ipython nbconvert mynotebook.ipynb which will convert mynotebook.ipynb to the default format (probably HTML). You can specify the export format with `--to`. Options include {0} > ipython nbconvert --to latex mynotebook.ipynb Both HTML and LaTeX support multiple output templates. LaTeX includes 'base', 'article' and 'report'. HTML includes 'basic' and 'full'. You can specify the flavor of the format used. > ipython nbconvert --to html --template basic mynotebook.ipynb You can also pipe the output to stdout, rather than a file > ipython nbconvert mynotebook.ipynb --stdout PDF is generated via latex > ipython nbconvert mynotebook.ipynb --to pdf You can get (and serve) a Reveal.js-powered slideshow > ipython nbconvert myslides.ipynb --to slides --post serve Multiple notebooks can be given at the command line in a couple of different ways: > ipython nbconvert notebook*.ipynb > ipython nbconvert notebook1.ipynb notebook2.ipynb or you can specify the notebooks list in a config file, containing:: c.NbConvertApp.notebooks = ["my_notebook.ipynb"] > ipython nbconvert --config mycfg.py """.format(get_export_names())) # Writer specific variables writer = Instance('IPython.nbconvert.writers.base.WriterBase', help="""Instance of the writer class used to write the results of the conversion.""") writer_class = DottedObjectName('FilesWriter', config=True, help="""Writer class used to write the results of the conversion""") writer_aliases = { 'fileswriter': 'IPython.nbconvert.writers.files.FilesWriter', 'debugwriter': 'IPython.nbconvert.writers.debug.DebugWriter', 'stdoutwriter': 'IPython.nbconvert.writers.stdout.StdoutWriter' } writer_factory = Type() def _writer_class_changed(self, name, old, new): if new.lower() in self.writer_aliases: new = self.writer_aliases[new.lower()] self.writer_factory = import_item(new) # Post-processor specific variables postprocessor = Instance( 'IPython.nbconvert.postprocessors.base.PostProcessorBase', help="""Instance of the PostProcessor class used to write the results of the conversion.""") postprocessor_class = DottedOrNone( config=True, help="""PostProcessor class used to write the results of the conversion""") postprocessor_aliases = { 'serve': 'IPython.nbconvert.postprocessors.serve.ServePostProcessor' } postprocessor_factory = Type() def _postprocessor_class_changed(self, name, old, new): if new.lower() in self.postprocessor_aliases: new = self.postprocessor_aliases[new.lower()] if new: self.postprocessor_factory = import_item(new) # Other configurable variables export_format = CaselessStrEnum(get_export_names(), default_value="html", config=True, help="""The export format to be used.""") notebooks = List([], config=True, help="""List of notebooks to convert. Wildcards are supported. Filenames passed positionally will be added to the list. """) @catch_config_error def initialize(self, argv=None): self.init_syspath() super(NbConvertApp, self).initialize(argv) self.init_notebooks() self.init_writer() self.init_postprocessor() def init_syspath(self): """ Add the cwd to the sys.path ($PYTHONPATH) """ sys.path.insert(0, os.getcwd()) def init_notebooks(self): """Construct the list of notebooks. If notebooks are passed on the command-line, they override notebooks specified in config files. Glob each notebook to replace notebook patterns with filenames. """ # Specifying notebooks on the command-line overrides (rather than adds) # the notebook list if self.extra_args: patterns = self.extra_args else: patterns = self.notebooks # Use glob to replace all the notebook patterns with filenames. filenames = [] for pattern in patterns: # Use glob to find matching filenames. Allow the user to convert # notebooks without having to type the extension. globbed_files = glob.glob(pattern) globbed_files.extend(glob.glob(pattern + '.ipynb')) if not globbed_files: self.log.warn("pattern %r matched no files", pattern) for filename in globbed_files: if not filename in filenames: filenames.append(filename) self.notebooks = filenames def init_writer(self): """ Initialize the writer (which is stateless) """ self._writer_class_changed(None, self.writer_class, self.writer_class) self.writer = self.writer_factory(parent=self) if hasattr(self.writer, 'build_directory') and self.writer.build_directory != '': self.use_output_suffix = False def init_postprocessor(self): """ Initialize the postprocessor (which is stateless) """ self._postprocessor_class_changed(None, self.postprocessor_class, self.postprocessor_class) if self.postprocessor_factory: self.postprocessor = self.postprocessor_factory(parent=self) def start(self): """ Ran after initialization completed """ super(NbConvertApp, self).start() self.convert_notebooks() def init_single_notebook_resources(self, notebook_filename): """Step 1: Initialize resources This intializes the resources dictionary for a single notebook. This method should return the resources dictionary, and MUST include the following keys: - profile_dir: the location of the profile directory - unique_key: the notebook name - output_files_dir: a directory where output files (not including the notebook itself) should be saved """ # Get a unique key for the notebook and set it in the resources object. basename = os.path.basename(notebook_filename) notebook_name = basename[:basename.rfind('.')] if self.output_base: # strip duplicate extension from output_base, to avoid Basname.ext.ext if getattr(self.exporter, 'file_extension', False): base, ext = os.path.splitext(self.output_base) if ext == self.exporter.file_extension: self.output_base = base notebook_name = self.output_base self.log.debug("Notebook name is '%s'", notebook_name) # first initialize the resources we want to use resources = {} resources['profile_dir'] = self.profile_dir.location resources['unique_key'] = notebook_name resources['output_files_dir'] = '%s_files' % notebook_name return resources def export_single_notebook(self, notebook_filename, resources): """Step 2: Export the notebook Exports the notebook to a particular format according to the specified exporter. This function returns the output and (possibly modified) resources from the exporter. """ try: output, resources = self.exporter.from_filename( notebook_filename, resources=resources) except ConversionException: self.log.error("Error while converting '%s'", notebook_filename, exc_info=True) self.exit(1) return output, resources def write_single_notebook(self, output, resources): """Step 3: Write the notebook to file This writes output from the exporter to file using the specified writer. It returns the results from the writer. """ if 'unique_key' not in resources: raise KeyError( "unique_key MUST be specified in the resources, but it is not") notebook_name = resources['unique_key'] if self.use_output_suffix and not self.output_base: notebook_name += resources.get('output_suffix', '') write_results = self.writer.write(output, resources, notebook_name=notebook_name) self.log.info(write_results) return write_results def postprocess_single_notebook(self, write_results): """Step 4: Postprocess the notebook This postprocesses the notebook after it has been written, taking as an argument the results of writing the notebook to file. This only actually does anything if a postprocessor has actually been specified. """ # Post-process if post processor has been defined. if hasattr(self, 'postprocessor') and self.postprocessor: self.postprocessor(write_results) def convert_single_notebook(self, notebook_filename): """Convert a single notebook. Performs the following steps: 1. Initialize notebook resources 2. Export the notebook to a particular format 3. Write the exported notebook to file 4. (Maybe) postprocess the written file """ self.log.info("Converting notebook %s to %s", notebook_filename, self.export_format) resources = self.init_single_notebook_resources(notebook_filename) output, resources = self.export_single_notebook( notebook_filename, resources) write_results = self.write_single_notebook(output, resources) self.postprocess_single_notebook(write_results) def convert_notebooks(self): """ Convert the notebooks in the self.notebook traitlet """ # check that the output base isn't specified if there is more than # one notebook to convert if self.output_base != '' and len(self.notebooks) > 1: self.log.error(""" UsageError: --output flag or `NbConvertApp.output_base` config option cannot be used when converting multiple notebooks. """) self.exit(1) self.log.info("Starting to convert_notebooks()") # initialize the exporter self.exporter = exporter_map[self.export_format](config=self.config) # no notebooks to convert! if len(self.notebooks) == 0: self.print_help() sys.exit(-1) # convert each notebook for notebook_filename in self.notebooks: self.convert_single_notebook(notebook_filename)
class NbConvertApp(BaseIPythonApplication): """Application used to convert to and from notebook file type (*.ipynb)""" name = 'ipython-nbconvert' aliases = nbconvert_aliases flags = nbconvert_flags def _log_level_default(self): return logging.INFO def _classes_default(self): classes = [NbConvertBase] for pkg in (exporters, transformers, writers): for name in dir(pkg): cls = getattr(pkg, name) if isinstance(cls, type) and issubclass(cls, Configurable): classes.append(cls) return classes description = Unicode( u"""This application is used to convert notebook files (*.ipynb) to various other formats. WARNING: THE COMMANDLINE INTERFACE MAY CHANGE IN FUTURE RELEASES.""") output_base = Unicode('', config=True, help='''overwrite base name use for output files. can only be use when converting one notebook at a time. ''') examples = Unicode(u""" The simplest way to use nbconvert is > ipython nbconvert mynotebook.ipynb which will convert mynotebook.ipynb to the default format (probably HTML). You can specify the export format with `--to`. Options include {0} > ipython nbconvert --to latex mynotebook.ipnynb Both HTML and LaTeX support multiple output templates. LaTeX includes 'basic', 'book', and 'article'. HTML includes 'basic' and 'full'. You can specify the flavor of the format used. > ipython nbconvert --to html --template basic mynotebook.ipynb You can also pipe the output to stdout, rather than a file > ipython nbconvert mynotebook.ipynb --stdout A post-processor can be used to compile a PDF > ipython nbconvert mynotebook.ipynb --to latex --post PDF You can get (and serve) a Reveal.js-powered slideshow > ipython nbconvert myslides.ipynb --to slides --post serve Multiple notebooks can be given at the command line in a couple of different ways: > ipython nbconvert notebook*.ipynb > ipython nbconvert notebook1.ipynb notebook2.ipynb or you can specify the notebooks list in a config file, containing:: c.NbConvertApp.notebooks = ["my_notebook.ipynb"] > ipython nbconvert --config mycfg.py """.format(get_export_names())) # Writer specific variables writer = Instance('IPython.nbconvert.writers.base.WriterBase', help="""Instance of the writer class used to write the results of the conversion.""") writer_class = DottedObjectName('FilesWriter', config=True, help="""Writer class used to write the results of the conversion""") writer_aliases = { 'fileswriter': 'IPython.nbconvert.writers.files.FilesWriter', 'debugwriter': 'IPython.nbconvert.writers.debug.DebugWriter', 'stdoutwriter': 'IPython.nbconvert.writers.stdout.StdoutWriter' } writer_factory = Type() def _writer_class_changed(self, name, old, new): if new.lower() in self.writer_aliases: new = self.writer_aliases[new.lower()] self.writer_factory = import_item(new) # Post-processor specific variables post_processor = Instance( 'IPython.nbconvert.post_processors.base.PostProcessorBase', help="""Instance of the PostProcessor class used to write the results of the conversion.""") post_processor_class = DottedOrNone( config=True, help="""PostProcessor class used to write the results of the conversion""") post_processor_aliases = { 'pdf': 'IPython.nbconvert.post_processors.pdf.PDFPostProcessor', 'serve': 'IPython.nbconvert.post_processors.serve.ServePostProcessor' } post_processor_factory = Type() def _post_processor_class_changed(self, name, old, new): if new.lower() in self.post_processor_aliases: new = self.post_processor_aliases[new.lower()] if new: self.post_processor_factory = import_item(new) # Other configurable variables export_format = CaselessStrEnum(get_export_names(), default_value="html", config=True, help="""The export format to be used.""") notebooks = List([], config=True, help="""List of notebooks to convert. Wildcards are supported. Filenames passed positionally will be added to the list. """) @catch_config_error def initialize(self, argv=None): super(NbConvertApp, self).initialize(argv) self.init_syspath() self.init_notebooks() self.init_writer() self.init_post_processor() def init_syspath(self): """ Add the cwd to the sys.path ($PYTHONPATH) """ sys.path.insert(0, os.getcwd()) def init_notebooks(self): """Construct the list of notebooks. If notebooks are passed on the command-line, they override notebooks specified in config files. Glob each notebook to replace notebook patterns with filenames. """ # Specifying notebooks on the command-line overrides (rather than adds) # the notebook list if self.extra_args: patterns = self.extra_args else: patterns = self.notebooks # Use glob to replace all the notebook patterns with filenames. filenames = [] for pattern in patterns: # Use glob to find matching filenames. Allow the user to convert # notebooks without having to type the extension. globbed_files = glob.glob(pattern) globbed_files.extend(glob.glob(pattern + '.ipynb')) if not globbed_files: self.log.warn("pattern %r matched no files", pattern) for filename in globbed_files: if not filename in filenames: filenames.append(filename) self.notebooks = filenames def init_writer(self): """ Initialize the writer (which is stateless) """ self._writer_class_changed(None, self.writer_class, self.writer_class) self.writer = self.writer_factory(parent=self) def init_post_processor(self): """ Initialize the post_processor (which is stateless) """ self._post_processor_class_changed(None, self.post_processor_class, self.post_processor_class) if self.post_processor_factory: self.post_processor = self.post_processor_factory(parent=self) def start(self): """ Ran after initialization completed """ super(NbConvertApp, self).start() self.convert_notebooks() def convert_notebooks(self): """ Convert the notebooks in the self.notebook traitlet """ # Export each notebook conversion_success = 0 if self.output_base != '' and len(self.notebooks) > 1: self.log.error( """UsageError: --output flag or `NbConvertApp.output_base` config option cannot be used when converting multiple notebooks. """) self.exit(1) exporter = exporter_map[self.export_format](config=self.config) for notebook_filename in self.notebooks: self.log.info("Converting notebook %s to %s", notebook_filename, self.export_format) # Get a unique key for the notebook and set it in the resources object. basename = os.path.basename(notebook_filename) notebook_name = basename[:basename.rfind('.')] if self.output_base: notebook_name = self.output_base resources = {} resources['unique_key'] = notebook_name resources['output_files_dir'] = '%s_files' % notebook_name self.log.info("Support files will be in %s", os.path.join(resources['output_files_dir'], '')) # Try to export try: output, resources = exporter.from_filename(notebook_filename, resources=resources) except ConversionException as e: self.log.error("Error while converting '%s'", notebook_filename, exc_info=True) self.exit(1) else: write_resultes = self.writer.write(output, resources, notebook_name=notebook_name) #Post-process if post processor has been defined. if hasattr(self, 'post_processor') and self.post_processor: self.post_processor(write_resultes) conversion_success += 1 # If nothing was converted successfully, help the user. if conversion_success == 0: self.print_help() sys.exit(-1)
class IPythonConsoleApp(Configurable): name = 'ipython-console-mixin' default_config_file_name='ipython_config.py' description = """ The IPython Mixin Console. This class contains the common portions of console client (QtConsole, ZMQ-based terminal console, etc). It is not a full console, in that launched terminal subprocesses will not be able to accept input. The Console using this mixing supports various extra features beyond the single-process Terminal IPython shell, such as connecting to existing kernel, via: ipython <appname> --existing as well as tunnel via SSH """ classes = classes flags = Dict(flags) aliases = Dict(aliases) kernel_manager_class = BlockingKernelManager kernel_argv = List(Unicode) # frontend flags&aliases to be stripped when building kernel_argv frontend_flags = Any(app_flags) frontend_aliases = Any(app_aliases) # create requested profiles by default, if they don't exist: auto_create = CBool(True) # connection info: transport = CaselessStrEnum(['tcp', 'ipc'], default_value='tcp', config=True) ip = Unicode(config=True, help="""Set the kernel\'s IP address [default localhost]. If the IP address is something other than localhost, then Consoles on other machines will be able to connect to the Kernel, so be careful!""" ) def _ip_default(self): if self.transport == 'tcp': return LOCALHOST else: # this can fire early if ip is given, # in which case our return value is meaningless if not hasattr(self, 'profile_dir'): return '' ipcdir = os.path.join(self.profile_dir.security_dir, 'kernel-%s' % os.getpid()) os.makedirs(ipcdir) atexit.register(lambda : shutil.rmtree(ipcdir)) return os.path.join(ipcdir, 'ipc') sshserver = Unicode('', config=True, help="""The SSH server to use to connect to the kernel.""") sshkey = Unicode('', config=True, help="""Path to the ssh key to use for logging in to the ssh server.""") hb_port = Int(0, config=True, help="set the heartbeat port [default: random]") shell_port = Int(0, config=True, help="set the shell (ROUTER) port [default: random]") iopub_port = Int(0, config=True, help="set the iopub (PUB) port [default: random]") stdin_port = Int(0, config=True, help="set the stdin (DEALER) port [default: random]") connection_file = Unicode('', config=True, help="""JSON file in which to store connection info [default: kernel-<pid>.json] This file will contain the IP, ports, and authentication key needed to connect clients to this kernel. By default, this file will be created in the security-dir of the current profile, but can be specified by absolute path. """) def _connection_file_default(self): return 'kernel-%i.json' % os.getpid() existing = CUnicode('', config=True, help="""Connect to an already running kernel""") confirm_exit = CBool(True, config=True, help=""" Set to display confirmation dialog on exit. You can always use 'exit' or 'quit', to force a direct exit without any confirmation.""", ) def build_kernel_argv(self, argv=None): """build argv to be passed to kernel subprocess""" if argv is None: argv = sys.argv[1:] self.kernel_argv = swallow_argv(argv, self.frontend_aliases, self.frontend_flags) # kernel should inherit default config file from frontend self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name) def init_connection_file(self): """find the connection file, and load the info if found. The current working directory and the current profile's security directory will be searched for the file if it is not given by absolute path. When attempting to connect to an existing kernel and the `--existing` argument does not match an existing file, it will be interpreted as a fileglob, and the matching file in the current profile's security dir with the latest access time will be used. After this method is called, self.connection_file contains the *full path* to the connection file, never just its name. """ if self.existing: try: cf = find_connection_file(self.existing) except Exception: self.log.critical("Could not find existing kernel connection file %s", self.existing) self.exit(1) self.log.info("Connecting to existing kernel: %s" % cf) self.connection_file = cf else: # not existing, check if we are going to write the file # and ensure that self.connection_file is a full path, not just the shortname try: cf = find_connection_file(self.connection_file) except Exception: # file might not exist if self.connection_file == os.path.basename(self.connection_file): # just shortname, put it in security dir cf = os.path.join(self.profile_dir.security_dir, self.connection_file) else: cf = self.connection_file self.connection_file = cf # should load_connection_file only be used for existing? # as it is now, this allows reusing ports if an existing # file is requested try: self.load_connection_file() except Exception: self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) self.exit(1) def load_connection_file(self): """load ip/port/hmac config from JSON connection file""" # this is identical to KernelApp.load_connection_file # perhaps it can be centralized somewhere? try: fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir]) except IOError: self.log.debug("Connection File not found: %s", self.connection_file) return self.log.debug(u"Loading connection file %s", fname) with open(fname) as f: cfg = json.load(f) self.transport = cfg.get('transport', 'tcp') if 'ip' in cfg: self.ip = cfg['ip'] for channel in ('hb', 'shell', 'iopub', 'stdin'): name = channel + '_port' if getattr(self, name) == 0 and name in cfg: # not overridden by config or cl_args setattr(self, name, cfg[name]) if 'key' in cfg: self.config.Session.key = str_to_bytes(cfg['key']) def init_ssh(self): """set up ssh tunnels, if needed.""" if not self.sshserver and not self.sshkey: return if self.transport != 'tcp': self.log.error("Can only use ssh tunnels with TCP sockets, not %s", self.transport) return if self.sshkey and not self.sshserver: # specifying just the key implies that we are connecting directly self.sshserver = self.ip self.ip = LOCALHOST # build connection dict for tunnels: info = dict(ip=self.ip, shell_port=self.shell_port, iopub_port=self.iopub_port, stdin_port=self.stdin_port, hb_port=self.hb_port ) self.log.info("Forwarding connections to %s via %s"%(self.ip, self.sshserver)) # tunnels return a new set of ports, which will be on localhost: self.ip = LOCALHOST try: newports = tunnel_to_kernel(info, self.sshserver, self.sshkey) except: # even catch KeyboardInterrupt self.log.error("Could not setup tunnels", exc_info=True) self.exit(1) self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports cf = self.connection_file base,ext = os.path.splitext(cf) base = os.path.basename(base) self.connection_file = os.path.basename(base)+'-ssh'+ext self.log.critical("To connect another client via this tunnel, use:") self.log.critical("--existing %s" % self.connection_file) def _new_connection_file(self): cf = '' while not cf: # we don't need a 128b id to distinguish kernels, use more readable # 48b node segment (12 hex chars). Users running more than 32k simultaneous # kernels can subclass. ident = str(uuid.uuid4()).split('-')[-1] cf = os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % ident) # only keep if it's actually new. Protect against unlikely collision # in 48b random search space cf = cf if not os.path.exists(cf) else '' return cf def init_kernel_manager(self): # Don't let Qt or ZMQ swallow KeyboardInterupts. signal.signal(signal.SIGINT, signal.SIG_DFL) # Create a KernelManager and start a kernel. self.kernel_manager = self.kernel_manager_class( transport=self.transport, ip=self.ip, shell_port=self.shell_port, iopub_port=self.iopub_port, stdin_port=self.stdin_port, hb_port=self.hb_port, connection_file=self.connection_file, config=self.config, ) # start the kernel if not self.existing: self.kernel_manager.start_kernel(extra_arguments=self.kernel_argv) elif self.sshserver: # ssh, write new connection file self.kernel_manager.write_connection_file() atexit.register(self.kernel_manager.cleanup_connection_file) self.kernel_manager.start_channels() def initialize(self, argv=None): """ Classes which mix this class in should call: IPythonConsoleApp.initialize(self,argv) """ self.init_connection_file() default_secure(self.config) self.init_ssh() self.init_kernel_manager()
class InteractiveShellApp(Configurable): """A Mixin for applications that start InteractiveShell instances. Provides configurables for loading extensions and executing files as part of configuring a Shell environment. The following methods should be called by the :meth:`initialize` method of the subclass: - :meth:`init_path` - :meth:`init_shell` (to be implemented by the subclass) - :meth:`init_gui_pylab` - :meth:`init_extensions` - :meth:`init_code` """ extensions = List( Unicode, config=True, help="A list of dotted module names of IPython extensions to load.") extra_extension = Unicode( '', config=True, help="dotted module name of an IPython extension to load.") def _extra_extension_changed(self, name, old, new): if new: # add to self.extensions self.extensions.append(new) # Extensions that are always loaded (not configurable) default_extensions = List(Unicode, [u'storemagic'], config=False) exec_files = List(Unicode, config=True, help="""List of files to run at IPython startup.""") file_to_run = Unicode('', config=True, help="""A file to be run""") exec_lines = List(Unicode, config=True, help="""lines of code to run at IPython startup.""") code_to_run = Unicode('', config=True, help="Execute the given command string.") module_to_run = Unicode('', config=True, help="Run the module as a script.") gui = CaselessStrEnum( ('qt', 'wx', 'gtk', 'glut', 'pyglet', 'osx'), config=True, help= "Enable GUI event loop integration ('qt', 'wx', 'gtk', 'glut', 'pyglet', 'osx')." ) pylab = CaselessStrEnum( ['tk', 'qt', 'wx', 'gtk', 'osx', 'inline', 'auto'], config=True, help="""Pre-load matplotlib and numpy for interactive use, selecting a particular matplotlib backend and loop integration. """) pylab_import_all = Bool( True, config=True, help="""If true, an 'import *' is done from numpy and pylab, when using pylab""") shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') def init_path(self): """Add current working directory, '', to sys.path""" if sys.path[0] != '': sys.path.insert(0, '') def init_shell(self): raise NotImplementedError("Override in subclasses") def init_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" if self.gui or self.pylab: shell = self.shell try: if self.pylab: gui, backend = pylabtools.find_gui_and_backend(self.pylab) self.log.info("Enabling GUI event loop integration, " "toolkit=%s, pylab=%s" % (gui, self.pylab)) shell.enable_pylab(gui, import_all=self.pylab_import_all, welcome_message=True) else: self.log.info("Enabling GUI event loop integration, " "toolkit=%s" % self.gui) shell.enable_gui(self.gui) except Exception: self.log.warn("GUI event loop or pylab initialization failed") self.shell.showtraceback() def init_extensions(self): """Load all IPython extensions in IPythonApp.extensions. This uses the :meth:`ExtensionManager.load_extensions` to load all the extensions listed in ``self.extensions``. """ try: self.log.debug("Loading IPython extensions...") extensions = self.default_extensions + self.extensions for ext in extensions: try: self.log.info("Loading IPython extension: %s" % ext) self.shell.extension_manager.load_extension(ext) except: self.log.warn("Error in loading extension: %s" % ext + "\nCheck your config files in %s" % self.profile_dir.location) self.shell.showtraceback() except: self.log.warn("Unknown error in loading extensions:") self.shell.showtraceback() def init_code(self): """run the pre-flight code, specified via exec_lines""" self._run_startup_files() self._run_exec_lines() self._run_exec_files() self._run_cmd_line_code() self._run_module() # flush output, so itwon't be attached to the first cell sys.stdout.flush() sys.stderr.flush() # Hide variables defined here from %who etc. self.shell.user_ns_hidden.update(self.shell.user_ns) def _run_exec_lines(self): """Run lines of code in IPythonApp.exec_lines in the user's namespace.""" if not self.exec_lines: return try: self.log.debug("Running code from IPythonApp.exec_lines...") for line in self.exec_lines: try: self.log.info("Running code in user namespace: %s" % line) self.shell.run_cell(line, store_history=False) except: self.log.warn("Error in executing line in user " "namespace: %s" % line) self.shell.showtraceback() except: self.log.warn("Unknown error in handling IPythonApp.exec_lines:") self.shell.showtraceback() def _exec_file(self, fname): try: full_filename = filefind(fname, [u'.', self.ipython_dir]) except IOError as e: self.log.warn("File not found: %r" % fname) return # Make sure that the running script gets a proper sys.argv as if it # were run from a system shell. save_argv = sys.argv sys.argv = [full_filename] + self.extra_args[1:] # protect sys.argv from potential unicode strings on Python 2: if not py3compat.PY3: sys.argv = [py3compat.cast_bytes(a) for a in sys.argv] try: if os.path.isfile(full_filename): if full_filename.endswith('.ipy'): self.log.info("Running file in user namespace: %s" % full_filename) self.shell.safe_execfile_ipy(full_filename) else: # default to python, even without extension self.log.info("Running file in user namespace: %s" % full_filename) # Ensure that __file__ is always defined to match Python behavior self.shell.user_ns['__file__'] = fname try: self.shell.safe_execfile(full_filename, self.shell.user_ns) finally: del self.shell.user_ns['__file__'] finally: sys.argv = save_argv def _run_startup_files(self): """Run files from profile startup directory""" startup_dir = self.profile_dir.startup_dir startup_files = glob.glob(os.path.join(startup_dir, '*.py')) startup_files += glob.glob(os.path.join(startup_dir, '*.ipy')) if not startup_files: return self.log.debug("Running startup files from %s...", startup_dir) try: for fname in sorted(startup_files): self._exec_file(fname) except: self.log.warn("Unknown error in handling startup files:") self.shell.showtraceback() def _run_exec_files(self): """Run files from IPythonApp.exec_files""" if not self.exec_files: return self.log.debug("Running files in IPythonApp.exec_files...") try: for fname in self.exec_files: self._exec_file(fname) except: self.log.warn("Unknown error in handling IPythonApp.exec_files:") self.shell.showtraceback() def _run_cmd_line_code(self): """Run code or file specified at the command-line""" if self.code_to_run: line = self.code_to_run try: self.log.info("Running code given at command line (c=): %s" % line) self.shell.run_cell(line, store_history=False) except: self.log.warn("Error in executing line in user namespace: %s" % line) self.shell.showtraceback() # Like Python itself, ignore the second if the first of these is present elif self.file_to_run: fname = self.file_to_run try: self._exec_file(fname) except: self.log.warn("Error in executing file in user namespace: %s" % fname) self.shell.showtraceback() def _run_module(self): """Run module specified at the command-line.""" if self.module_to_run: # Make sure that the module gets a proper sys.argv as if it were # run using `python -m`. save_argv = sys.argv sys.argv = [sys.executable] + self.extra_args try: self.shell.safe_run_module(self.module_to_run, self.shell.user_ns) finally: sys.argv = save_argv
class Knitpy(LoggingConfigurable): """Engine used to convert from python markdown (``*.pymd``) to html/latex/...""" keep_md = Bool(False, config=True, help="""Whether to keep the temporary md files""") log_to_file = Bool(False, config=True, help="""Whether to send the log to a file""") extra_document_configs = List( default_value=[], config=True, help="Additional configurations for FinalOutputDocuments") default_export_format = CaselessStrEnum( VALID_OUTPUT_FORMAT_NAMES, default_value=DEFAULT_OUTPUT_FORMAT_NAME, config=True, help= """The export format to be used (can't by from extra_document_configs!).""" ) kernel_debug = Bool( False, config=True, help="""Whether to output kernel messages to the (debug) log""") timeout = Integer(10, config=True, help="timeout for individual code executions") # Things for the parser... chunk_begin = CRegExpMultiline( r'^\s*```+\s*{[.]?(?P<engine>[a-z]+)\s*(?P<args>.*)}\s*$', config=True, help="chunk begin regex (must include the named " "group 'engine' and 'args'") chunk_end = CRegExpMultiline(r'^\s*```+\s*$', config=True, help="chunk end regex") inline_code = CRegExpMultiline( r'`(?P<engine>[a-z]+) +([^`]+)\s*`', config=True, help="inline code regex (must include a named group 'engine')") comment_line = CRegExp(r'^\s*#', config=True, help="comment line regex") yaml_separator = CRegExpMultiline(r"^---\s*$", config=True, help="separator for the yaml metadata") def __init__(self, **kwargs): super(Knitpy, self).__init__(**kwargs) self.init_kernel_manager() self.init_engines() self.init_output_configurations() def init_kernel_manager(self): self._km = MultiKernelManager(log=self.log, parent=self) self._ksm = KernelSpecManager(log=self.log, parent=self) self._kernels = {} #ksm.find_kernel_specs() def init_engines(self): self._engines = {} self._engines["python"] = PythonKnitpyEngine(parent=self) # TODO: check that every kernel_name is in ksm.find_kernel_specs() def init_output_configurations(self): self._outputs = {} for config in DEFAULT_FINAL_OUTPUT_FORMATS: fod = FinalOutputConfiguration(parent=self, **config) self._outputs[config["name"]] = fod self._outputs[config["alias"]] = fod for config in self.extra_document_configs: fod = FinalOutputConfiguration(parent=self, **config) self._outputs[config["name"]] = fod self._outputs[config["alias"]] = fod def parse_document(self, input): if os.path.exists(input): filename = input f = codecs.open(filename, 'r', 'UTF-8') doc = f.read() else: doc = input filename = "anonymous_input" # the yaml can stay in the doc, pandoc will remove '---' blocks # pandoc will also do it's own interpretation and use title/author and so on... # ToDo: not sure of that should stay or if we should start with clean metadata # title, author, date # title: "A first try" # author: "Jan Schulz" # date: "Monday, February 23, 2015" # default values metadata = { "title": filename, "author": getpass.getuser(), "date": datetime.datetime.now().strftime("%A, %B %d, %Y") } pos = 0 start = self.yaml_separator.search(doc, pos) if not start is None: end = self.yaml_separator.search(doc, start.end()) if end is None: raise ParseException("Found no metadata end separator.") try: res = yaml.load(doc[start.end():end.start()]) self.log.debug("Metadata: %s", res) metadata.update(res) except Exception as e: raise ParseException("Malformed metadata: %s" % str(e)) parsed_doc = self._parse_blocks(doc) return parsed_doc, metadata def _parse_blocks(self, doc): result = [] doc_pos = 0 blocks = self.chunk_begin.finditer(doc) for block_start in blocks: # process the text before the match text = doc[doc_pos:block_start.start()] self._parse_inline(text, result) # TODO: somehow a empty line before a codeblock vanishes, so add one here result.append((TTEXT, "\n")) # now the block itself # find the end of the block block_end = self.chunk_end.search(doc, block_start.end()) if block_end is None: raise ParseException( "Found no end for the block starting at pos %s" % block_start.end()) result.append((TBLOCK, (doc[block_start.end():block_end.start()], block_start.groupdict()))) doc_pos = block_end.end() # text after the last block self._parse_inline(doc[doc_pos:], result) return result def _parse_inline(self, text, result): text_pos = 0 for inline in self.inline_code.finditer(text): # text before inline code result.append((TTEXT, text[text_pos:inline.start()])) # inline code engine_offset = len(inline.group('engine')) + 1 result.append( (TINLINE, (text[inline.start() + engine_offset + 1:inline.end() - 1], inline.groupdict()))) text_pos = inline.end() # text after the last inline code result.append((TTEXT, text[text_pos:])) def _all_lines_comments(self, lines): for line in lines.split("\n"): if not self.comment_line.match(line): return False return True def convert(self, parsed, output): context = ExecutionContext(output=output) for entry in parsed: if entry[0] == TBLOCK: context.mode = "block" self._process_code(entry[1], context=context) elif entry[0] == TINLINE: context.mode = "inline" self._process_code(entry[1], context=context) elif entry[0] == TTEXT: output.add_text(entry[1]) else: raise ParseException("Found something unexpected: %s" % entry) # process_code opened kernels, so close them here self._km.shutdown_all() # workaround for https://github.com/ipython/ipython/issues/8007 # FIXME: remove if IPython >3.0 is in require self._km._kernels.clear() self._kernels = {} return output def _process_code(self, input, context): context.execution_started() # setup the execution context code = input[0] intro = input[1] engine_name = intro["engine"] raw_args = intro.get("args", "") args = self._parse_args(raw_args) # for compatibility with knitr, where python is specified via "{r engine='python'}" if "engine" in args: engine_name = args.pop("engine") self.log.debug("Running on engine: %s", engine_name) try: engine = self._engines[engine_name] except: raise ParseException("Unknown codeblock type: %s" % engine_name) assert not engine is None, "Engine is None" context.engine = engine if not engine.name in context.enabled_documents: plotting_formats = context.output.export_config.accepted_image_formats plot_code = engine.get_plotting_format_code(plotting_formats) self._run_silently(context.engine.kernel, plot_code) context.enabled_documents.append(engine.name) self.log.info("Enabled image formats '%s' in engine '%s'.", plotting_formats, engine.name) # configure the context if "echo" in args: context.echo = args.pop("echo") # eval=False means that we don't execute the block at all if "eval" in args: _eval = args.pop("eval") if _eval is False: if context.echo: code = code.replace(os.linesep, "\n").lstrip("\n") context.output.add_code(code, language=engine.language) return if "results" in args: context.results = args.pop("results") if "include" in args: context.include = args.pop("include") if "chunk_label" in args: context.chunk_label = args.pop("chunk_label") else: context.chunk_label = u"unnamed-chunk-%s" % context.chunk_number if "comment" in args: context.comment = args.pop("comment") if args: self.log.debug("Found unhandled args: %s", args) lines = '' code_lines = code.split('\n') space_re = re.compile(r'^([\s]+)') spaces = [] # TODO: this whole "merge with the next line" should be rewritten as a generator def loop_continues(line_no): if len(code_lines) <= line_no: return False candidate = code_lines[line_no] # comments should be swallowed if a line further down has code in it with the # right number of spaces in front while candidate.strip() == "" or self._all_lines_comments( candidate): line_no += 1 if len(code_lines) <= line_no: return False candidate = code_lines[line_no] # the next code line must have either the same number of spaces (still in a loop), # or less spaces as in 'spaces' (nested loop) or none (end of loop). If more spaces # are found or different types of spaces, this will result in an error which will be # shown when the code is executed... while spaces: possible_space = spaces[-1] if candidate[:len(possible_space)] == possible_space: # ok, we are at the "right" level of space return True # not our "space", so remove it and try the one one nesting above spaces.pop() return False for line_no in range(len(code_lines)): cur_line = code_lines[line_no] lines = lines + cur_line # check if we are in a loop and if so, if the next line also belongs to this loop # this only catches the case where we are *in* a loop and not the loop start (the line # with a ':' in it. That line is catched by the is_complete call below. nested loops # are also catched due to the space in front of it m = space_re.match(cur_line) if m: cur_space = m.group(0) spaces.append(cur_space) if loop_continues(line_no + 1): lines += "\n" continue if spaces: # we are in a loop, as spaces has some spaces in it, but the code above didn't find # any spaces in front of the line -> this is the case when loop_continues found a # new codeline from this loop after a comment with different spaces in front of # it or an empty line. This could be such an empty/comment line and we have to # look at the next line as well! if cur_line.strip() == "" or self._all_lines_comments( cur_line): lines += "\n" continue # we have a block of code, including all lines of a loop msg = engine.kernel.is_complete(lines + "\n\n") reply = engine.kernel.get_shell_msg(timeout=self.timeout) assert reply['msg_type'] == 'is_complete_reply', str(reply) if self.kernel_debug: self.log.debug("completion_request: %s", msg) if reply['content']['status'] == 'complete': if lines.strip() == "": # No requests for "no code" lines = "" continue elif self._all_lines_comments(lines): # comments should go to to the next code block lines += "\n" continue # run the lines self._run_lines(lines + "\n", context) lines = "" elif reply['content']['status'] == 'invalid': # TODO: not sure how this should be handled # Either abort execution of the whole file or just retry with the next line? # However this should be handled via a user message self.log.info("Code invalid:\n%s", lines) context.output.add_code(lines, language=engine.language) context.output.add_execution_error("Code invalid") lines = "" else: # the "incomplete" case: don't run anything wait for the next line lines += "\n" # This can only happen if the last line is incomplete # This will always result in an error! if lines.strip() != "": self._run_lines(lines, context) context.execution_finished() def _parse_args(self, raw_args): # Todo: knitr interprets all values, so code references are possible # This also means that we have to do args parsing at interpretation time, so that # variable from other code can be taken into account.. args = {} if raw_args.strip() == "": return args converter = { "True": True, "False": False, "None": None, "T": True, # Rs True/False "F": False, "TRUE": True, "FALSE": False, # treat Rs NA as None, probably overkill to look for float("NA") "NA": None, "NULL": None, } # The first is special as that can be the name of the chunk first = True for arg in raw_args.split(","): arg = arg.strip() if not "=" in arg: if not first: raise ParseException( "Malformed options for code chunk: '%s' in '%s'" % (arg, raw_args)) args["chunk_label"] = arg continue first = False label, value = arg.split("=") v = value.strip() # convert to real types. # TODO: Should be done by submitting the whole thing to the kernel, like knitr does # -> variables form one codecell can be used in the args of the next one ... if (v[0] == '"' and v[-1] == '"'): v = v[1:-1] elif (v[0] == "'" and v[-1] == "'"): v = v[1:-1] elif v in converter: v = converter[v] else: try: v = int(v) except: self.log.error( "Could not decode option value: '%s=%s'. Discarded...", label, v) continue args[label.strip()] = v return args def _run_lines(self, lines, context): kernel = context.engine.kernel msg_id = kernel.execute(lines) if self.kernel_debug: self.log.debug("Executing lines (msg_id=%s):\n%s", msg_id, lines) # wait for finish, with timeout # At first we have to wait until the kernel tells us it is finished with running the code while True: try: msg = kernel.shell_channel.get_msg(timeout=self.timeout) if self.kernel_debug: self.log.debug("shell msg: %s", msg) except Empty: # This indicates that something bad happened, as AFAIK this should return... self.log.error("Timeout waiting for execute reply") raise KnitpyException("Timeout waiting for execute reply.") if msg['parent_header'].get('msg_id') == msg_id: # It's finished, and we got our reply, so next look at the results break else: # not our reply self.log.debug( "Discarding message from a different client: %s" % msg) continue # Now look at the results of our code execution and earlier completion requests # We handle messages until the kernel indicates it's ide again status_idle_again = False while True: try: msg = kernel.get_iopub_msg(timeout=self.timeout) except Empty: # There should be at least some messages: we just executed code! # The only valid time could be when the timeout happened too early (aka long # running code in the document) -> we handle that below self.log.warn("Timeout waiting for expected IOPub output") break if msg['parent_header'].get('msg_id') != msg_id: if msg['parent_header'].get( u'msg_type') != u'is_complete_request': # not an output from our execution and not one of the complete_requests self.log.debug( "Discarding output from a different client: %s" % msg) else: # complete_requests are ok pass continue # Here we have some message which corresponds to our code execution msg_type = msg['msg_type'] content = msg['content'] # The kernel indicates some status: executing -> idle if msg_type == 'status': if content['execution_state'] == 'idle': # When idle, the kernel has executed all input status_idle_again = True break else: # the "starting execution" messages continue elif msg_type == 'clear_output': # we don't handle that!? self.log.debug( "Discarding unexpected 'clear_output' message: %s" % msg) continue ## So, from here on we have a messages with real content if self.kernel_debug: self.log.debug("iopub msg (%s): %s", msg_type, msg) if context.include: self._handle_return_message(msg, context) if not status_idle_again: self.log.error( "Code lines didn't execute in time. Don't use long-running code in " "documents or increase the timeout!") self.log.error("line(s): %s" % lines) def _handle_return_message(self, msg, context): if context.mode == "inline": #self.log.debug("inline: %s" % msg) if msg["msg_type"] == "execute_result": context.output.add_text(_plain_text(msg["content"])) elif context.mode == "block": #self.log.debug("block: %s" % msg) type = msg["msg_type"] if type == "execute_input": if context.echo: context.output.add_code(_code(msg[u'content']), language=context.engine.language) elif type == "stream": # {u'text': u'a\nb\nc\n', u'name': u'stdout'} # TODO: format stdout and stderr differently? txt = msg["content"].get("text", "") if txt.strip() == "": return if context.results == 'markup': context.output.add_output(txt) elif context.results == 'asis': context.output.add_asis(txt) elif context.results == 'hide': return else: # TODO: implement a caching system... again... self.log.warn( "Can't handle results='hold' yet, falling back to 'markup'." ) context.output.add_output(txt) elif (type == "execute_result") or (type == "display_data"): if context.results == 'hide': return if context.results == 'hold': self.log.warn( "Can't handle results='hold' yet, falling back to 'markup'." ) # Here we handle the output from the IPython display framework. # 1. If a object has a _display_ipython(), that will be called. This method should # publish (one) display_data message and return -> the content ends up in # "display_data" msg and the "executive_result" has no data # 2. else try different IPython.core.formatters for the object, which basically # call the right _repr_<whatever>_ method to get a formated string in that # mimetype. This is added as alternatives under content.data of the # "executive_result". # data has/can have multiple types of the same message data = msg[u"content"][u'data'] #self.log.debug(str(data)) # handle plots #self.log.debug("Accepted image mimetypes: %s", context.output.export_config.accepted_image_mimetypes) for mime_type in context.output.export_config.accepted_image_mimetypes: mime_data = data.get(mime_type, None) if mime_data is None: self.log.debug("No image found: %s", mime_type) continue try: self.log.debug("Trying to include image...") context.output.add_image(mime_type, mime_data, title="") except KnitpyOutputException as e: self.log.info("Couldn't include image: %s", e) continue return # now try some marked up text formats for mime_type in context.output.markup_mimetypes: mime_data = data.get(mime_type, None) if mime_data is None: continue try: self.log.debug("Trying to include markup text...") context.output.add_markup_text(mime_type, mime_data) except KnitpyOutputException as e: self.log.info("Couldn't include markup text: %s", e) continue return # as a last resort, try plain text... if u'text/plain' in data: txt = data.get(u"text/plain", "") if txt != "": if context.results == 'markup': context.output.add_output(txt) if txt[-1] != "\n": context.output.add_output("\n") elif context.results == 'asis': context.output.add_asis(txt) if txt[-1] != "\n": context.output.add_asis("\n") return # If we are here, we couldn't handle any of the more specific data types # and didn't find any output text excuse = "\n(Found data of type '{}', but couldn't handle it)\n" context.output.add_output(excuse.format(data.keys())) elif (type == "error"): ename = msg["content"].get("ename", "unknown exception") evalue = msg["content"].get("evalue", "unknown exception value") tb = msg["content"].get("traceback", "<not available>") if not is_string(tb): # remove the first line... tb = "\n".join(tb[1:]) self.log.info(tb) #there are ansi escape sequences in the traceback, which kills pandoc :-( if u"\x1b[1;32m" in tb: tb = "!! traceback unavailable due to included color sequences;\n" \ "!! execute `%colors NoColor` once before this line to remove them!" context.output.add_execution_error("%s: %s" % (ename, evalue), tb) else: self.log.debug("Ignored msg of type %s" % type) def _run_silently(self, kc, lines): try: msg_id = kc.execute(lines + "\n\n", silent=self.kernel_debug, store_history=False) self.log.debug("Executed silent code: %s", lines) reply = kc.get_shell_msg(timeout=self.timeout) assert reply['parent_header'].get( 'msg_id') == msg_id, "Wrong reply! " + str(reply) if self.kernel_debug: self.log.debug("Silent code shell reply: %s", reply) except Empty: self.log.error("Code took too long:\n %s", lines) # now empty the iopub channel (there is at least a "starting" message) while True: try: msg = kc.get_iopub_msg(timeout=0.1) if self.kernel_debug: self.log.debug("Silent code iopub msg: %s", msg) except Empty: break def _get_kernel(self, engine): kernel_name = engine.kernel_name kernel_startup_lines = engine.startup_lines if not kernel_name in self._kernels: self.log.info("Starting a new kernel: %s" % kernel_name) kernelid = self._km.start_kernel(kernel_name=kernel_name) #km.list_kernel_ids() kn = self._km.get_kernel(kernelid) kc = kn.client() self._kernels[kernel_name] = kc # now initalize the channels kc.start_channels() kc.wait_for_ready() self._run_silently(kc, kernel_startup_lines) self.log.info("Executed kernel startup lines for engine '%s'.", engine.name) return self._kernels[kernel_name] def get_output_format(self, fmt_name, config=None): self._ensure_valid_output(fmt_name) fod = self._outputs.get(fmt_name).copy() # self.log.info("%s: %s", fmt_name, config) if not config: pass elif isinstance(config, dict): fod.update(**config) elif config == "default": # html_document: default pass else: self.log.error( "Unknown config for document '%s': '%s'. Ignored...", fmt_name, config) return fod def _knit(self, input, outputdir_name, final_format="html", config=None): """Internal function to aid testing""" parsed, metadata = self.parse_document(input) # sets kpydoc.parsed and final_format = self.get_output_format(final_format, config=config) md_temp = TemporaryOutputDocument(fileoutputs=outputdir_name, export_config=final_format, log=self.log, parent=self) # get the temporary md file self.convert(parsed, md_temp) return md_temp.content def render(self, filename, output=None): """ Convert the filename to the given output format(s) """ # Export each documents conversion_success = 0 converted_docs = [] # save here to change back after the conversation. orig_cwd = os.getcwd() needs_chdir = False # expand $HOME and so on... filename = expand_path(filename) filename = os.path.abspath(filename) self.log.info("Converting %s..." % filename) basedir = os.path.dirname(filename) basename = os.path.splitext(os.path.basename(filename))[0] # It's easier if we just change wd to the dir of the file if unicode_type(basedir) != py3compat.getcwd(): os.chdir(basedir) needs_chdir = True self.log.info("Changing to working dir: %s" % basedir) filename = os.path.basename(filename) outputdir_name = os.path.splitext(basename)[0] + "_files" # parse the input document parsed, metadata = self.parse_document(filename) # get the output formats # order: kwarg overwrites default overwrites document output_formats = [self._outputs[self.default_export_format]] if output is None: self.log.debug("Converting to default output format [%s]!" % self.default_export_format) elif output == "all": outputs = metadata.get("output", None) # if nothing is specified, we keep the default if outputs is None: self.log.debug( "Did not find any specified output formats: using only default!" ) else: output_formats = [] for fmt_name, config in iteritems(outputs): fod = self.get_output_format(fmt_name, config) output_formats.append(fod) self.log.debug( "Converting to all specified output formats: %s" % [fmt.name for fmt in output_formats]) else: self._ensure_valid_output(output) output_formats = [self._outputs[output]] for final_format in output_formats: self.log.info("Converting document %s to %s", filename, final_format.name) # TODO: build a proper way to specify final output... md_temp = TemporaryOutputDocument(fileoutputs=outputdir_name, export_config=final_format, log=self.log, parent=self) # get the temporary md file self.convert(parsed, md_temp) if final_format.keep_md or self.keep_md: mdfilename = basename + "." + final_format.name + ".md" self.log.info("Saving the temporary markdown as '%s'." % mdfilename) # TODO: remove the first yaml metadata block and # put "#<title>\n<author>\n<date>" before the rest with codecs.open(mdfilename, 'w+b', 'UTF-8') as f: f.write(md_temp.content) # convert the md file to the final filetype input_format = "markdown" \ "+autolink_bare_uris" \ "+ascii_identifiers" \ "+tex_math_single_backslash-implicit_figures" \ "+fenced_code_attributes" extra = [ "--smart", # typographically correct output (curly quotes, etc) "--email-obfuscation", "none", #do not obfuscation email names with javascript "--self-contained", # include img/scripts as data urls "--standalone", # html with header + footer "--section-divs", ] outfilename = basename + "." + final_format.file_extension # exported is irrelevant, as we pass in a filename exported = pandoc(source=md_temp.content, to=final_format.pandoc_export_format, format=input_format, extra_args=extra, outputfile=outfilename) self.log.info("Written final output: %s" % outfilename) converted_docs.append(os.path.join(basedir, outfilename)) if needs_chdir: os.chdir(orig_cwd) return converted_docs def _ensure_valid_output(self, fmt_name): if fmt_name in self._outputs: return raise KnitpyException("Format '%s' is not a valid output format!" % fmt_name)