class EmbeddedSphinxShell(object): """An embedded IPython instance to run inside Sphinx""" def __init__(self): self.cout = StringIO() # Create config object for IPython config = Config() config.Global.display_banner = False config.Global.exec_lines = ['import numpy as np', 'from pylab import *' ] config.InteractiveShell.autocall = False config.InteractiveShell.autoindent = False config.InteractiveShell.colors = 'NoColor' config.InteractiveShell.cache_size = 0 # create a profile so instance history isn't saved tmp_profile_dir = tempfile.mkdtemp(prefix='profile_') profname = 'auto_profile_sphinx_build' pdir = os.path.join(tmp_profile_dir,profname) profile = ProfileDir.create_profile_dir(pdir) # Create and initialize ipython, but don't start its mainloop IP = InteractiveShell.instance(config=config, profile_dir=profile) # io.stdout redirect must be done *after* instantiating InteractiveShell io.stdout = self.cout io.stderr = self.cout # For debugging, so we can see normal output, use this: #from IPython.utils.io import Tee #io.stdout = Tee(self.cout, channel='stdout') # dbg #io.stderr = Tee(self.cout, channel='stderr') # dbg # Store a few parts of IPython we'll need. self.IP = IP self.user_ns = self.IP.user_ns self.user_global_ns = self.IP.user_global_ns self.input = '' self.output = '' self.is_verbatim = False self.is_doctest = False self.is_suppress = False # on the first call to the savefig decorator, we'll import # pyplot as plt so we can make a call to the plt.gcf().savefig self._pyplot_imported = False def clear_cout(self): self.cout.seek(0) self.cout.truncate(0) def process_input_line(self, line, store_history=True): """process the input, capturing stdout""" #print "input='%s'"%self.input stdout = sys.stdout splitter = self.IP.input_splitter try: sys.stdout = self.cout splitter.push(line) more = splitter.push_accepts_more() if not more: source_raw = splitter.source_raw_reset()[1] self.IP.run_cell(source_raw, store_history=store_history) finally: sys.stdout = stdout def process_image(self, decorator): """ # build out an image directive like # .. image:: somefile.png # :width 4in # # from an input like # savefig somefile.png width=4in """ savefig_dir = self.savefig_dir source_dir = self.source_dir saveargs = decorator.split(' ') filename = saveargs[1] # insert relative path to image file in source outfile = os.path.relpath(os.path.join(savefig_dir,filename), source_dir) imagerows = ['.. image:: %s'%outfile] for kwarg in saveargs[2:]: arg, val = kwarg.split('=') arg = arg.strip() val = val.strip() imagerows.append(' :%s: %s'%(arg, val)) image_file = os.path.basename(outfile) # only return file name image_directive = '\n'.join(imagerows) return image_file, image_directive # Callbacks for each type of token def process_input(self, data, input_prompt, lineno): """Process data block for INPUT token.""" decorator, input, rest = data image_file = None image_directive = None #print 'INPUT:', data # dbg is_verbatim = decorator=='@verbatim' or self.is_verbatim is_doctest = decorator=='@doctest' or self.is_doctest is_suppress = decorator=='@suppress' or self.is_suppress is_okexcept = decorator=='@okexcept' or self.is_okexcept is_savefig = decorator is not None and \ decorator.startswith('@savefig') def _remove_first_space_if_any(line): return line[1:] if line.startswith(' ') else line input_lines = lmap(_remove_first_space_if_any, input.split('\n')) self.datacontent = data continuation = ' %s: '%''.join(['.']*(len(str(lineno))+2)) if is_savefig: image_file, image_directive = self.process_image(decorator) ret = [] is_semicolon = False store_history = True for i, line in enumerate(input_lines): if line.endswith(';'): is_semicolon = True if is_semicolon or is_suppress: store_history = False if i==0: # process the first input line if is_verbatim: self.process_input_line('') self.IP.execution_count += 1 # increment it anyway else: # only submit the line in non-verbatim mode self.process_input_line(line, store_history=store_history) formatted_line = '%s %s'%(input_prompt, line) else: # process a continuation line if not is_verbatim: self.process_input_line(line, store_history=store_history) formatted_line = '%s%s'%(continuation, line) if not is_suppress: ret.append(formatted_line) if not is_suppress: if len(rest.strip()): if is_verbatim: # the "rest" is the standard output of the # input, which needs to be added in # verbatim mode ret.append(rest) self.cout.seek(0) output = self.cout.read() if not is_suppress and not is_semicolon: ret.append(output.decode('utf-8')) if not is_okexcept and "Traceback" in output: sys.stdout.write(output) self.cout.truncate(0) return (ret, input_lines, output, is_doctest, image_file, image_directive) #print 'OUTPUT', output # dbg def process_output(self, data, output_prompt, input_lines, output, is_doctest, image_file): """Process data block for OUTPUT token.""" if is_doctest: submitted = data.strip() found = output if found is not None: found = found.strip() # XXX - fperez: in 0.11, 'output' never comes with the prompt # in it, just the actual output text. So I think all this code # can be nuked... # the above comment does not appear to be accurate... (minrk) ind = found.find(output_prompt) if ind<0: e='output prompt="%s" does not match out line=%s' % \ (output_prompt, found) raise RuntimeError(e) found = found[len(output_prompt):].strip() if found!=submitted: e = ('doctest failure for input_lines="%s" with ' 'found_output="%s" and submitted output="%s"' % (input_lines, found, submitted) ) raise RuntimeError(e) #print 'doctest PASSED for input_lines="%s" with found_output="%s" and submitted output="%s"'%(input_lines, found, submitted) def process_comment(self, data): """Process data fPblock for COMMENT token.""" if not self.is_suppress: return [data] def save_image(self, image_file): """ Saves the image file to disk. """ self.ensure_pyplot() command = ('plt.gcf().savefig("%s", bbox_inches="tight", ' 'dpi=100)' % image_file) #print 'SAVEFIG', command # dbg self.process_input_line('bookmark ipy_thisdir', store_history=False) self.process_input_line('cd -b ipy_savedir', store_history=False) self.process_input_line(command, store_history=False) self.process_input_line('cd -b ipy_thisdir', store_history=False) self.process_input_line('bookmark -d ipy_thisdir', store_history=False) self.clear_cout() def process_block(self, block): """ process block from the block_parser and return a list of processed lines """ ret = [] output = None input_lines = None lineno = self.IP.execution_count input_prompt = self.promptin%lineno output_prompt = self.promptout%lineno image_file = None image_directive = None for token, data in block: if token==COMMENT: out_data = self.process_comment(data) elif token==INPUT: (out_data, input_lines, output, is_doctest, image_file, image_directive) = \ self.process_input(data, input_prompt, lineno) elif token==OUTPUT: out_data = \ self.process_output(data, output_prompt, input_lines, output, is_doctest, image_file) if out_data: ret.extend(out_data) # save the image files if image_file is not None: self.save_image(image_file) return ret, image_directive def ensure_pyplot(self): if self._pyplot_imported: return self.process_input_line('import matplotlib.pyplot as plt', store_history=False) def process_pure_python(self, content): """ content is a list of strings. it is unedited directive conent This runs it line by line in the InteractiveShell, prepends prompts as needed capturing stderr and stdout, then returns the content as a list as if it were ipython code """ output = [] savefig = False # keep up with this to clear figure multiline = False # to handle line continuation fmtin = self.promptin for lineno, line in enumerate(content): line_stripped = line.strip() if not len(line): output.append(line) # preserve empty lines in output continue # handle decorators if line_stripped.startswith('@'): output.extend([line]) if 'savefig' in line: savefig = True # and need to clear figure continue # handle comments if line_stripped.startswith('#'): output.extend([line]) continue # deal with multilines if not multiline: # not currently on a multiline if line_stripped.endswith('\\'): # now we are multiline = True cont_len = len(str(lineno)) + 2 line_to_process = line.strip('\\') output.extend([u("%s %s") % (fmtin%lineno,line)]) continue else: # no we're still not line_to_process = line.strip('\\') else: # we are currently on a multiline line_to_process += line.strip('\\') if line_stripped.endswith('\\'): # and we still are continuation = '.' * cont_len output.extend([(u(' %s: ')+line_stripped) % continuation]) continue # else go ahead and run this multiline then carry on # get output of line self.process_input_line(compat.text_type(line_to_process.strip()), store_history=False) out_line = self.cout.getvalue() self.clear_cout() # clear current figure if plotted if savefig: self.ensure_pyplot() self.process_input_line('plt.clf()', store_history=False) self.clear_cout() savefig = False # line numbers don't actually matter, they're replaced later if not multiline: in_line = u("%s %s") % (fmtin%lineno,line) output.extend([in_line]) else: output.extend([(u(' %s: ')+line_stripped) % continuation]) multiline = False if len(out_line): output.extend([out_line]) output.extend([u('')]) return output def process_pure_python2(self, content): """ content is a list of strings. it is unedited directive conent This runs it line by line in the InteractiveShell, prepends prompts as needed capturing stderr and stdout, then returns the content as a list as if it were ipython code """ output = [] savefig = False # keep up with this to clear figure multiline = False # to handle line continuation multiline_start = None fmtin = self.promptin ct = 0 # nuke empty lines content = [line for line in content if len(line.strip()) > 0] for lineno, line in enumerate(content): line_stripped = line.strip() if not len(line): output.append(line) continue # handle decorators if line_stripped.startswith('@'): output.extend([line]) if 'savefig' in line: savefig = True # and need to clear figure continue # handle comments if line_stripped.startswith('#'): output.extend([line]) continue continuation = u(' %s:')% ''.join(['.']*(len(str(ct))+2)) if not multiline: modified = u("%s %s") % (fmtin % ct, line_stripped) output.append(modified) ct += 1 try: ast.parse(line_stripped) output.append(u('')) except Exception: multiline = True multiline_start = lineno else: modified = u('%s %s') % (continuation, line) output.append(modified) try: ast.parse('\n'.join(content[multiline_start:lineno+1])) if (lineno < len(content) - 1 and _count_indent(content[multiline_start]) < _count_indent(content[lineno + 1])): continue output.extend([continuation, u('')]) multiline = False except Exception: pass continue return output
class EmbeddedSphinxShell(object): """An embedded IPython instance to run inside Sphinx""" def __init__(self, exec_lines=None, state=None): self.cout = StringIO() if exec_lines is None: exec_lines = [] self.state = state # Create config object for IPython config = Config() config.InteractiveShell.autocall = False config.InteractiveShell.autoindent = False config.InteractiveShell.colors = "NoColor" # create a profile so instance history isn't saved tmp_profile_dir = tempfile.mkdtemp(prefix="profile_") profname = "auto_profile_sphinx_build" pdir = os.path.join(tmp_profile_dir, profname) profile = ProfileDir.create_profile_dir(pdir) # Create and initialize global ipython, but don't start its mainloop. # This will persist across different EmbededSphinxShell instances. IP = InteractiveShell.instance(config=config, profile_dir=profile) # io.stdout redirect must be done after instantiating InteractiveShell io.stdout = self.cout io.stderr = self.cout # For debugging, so we can see normal output, use this: # from IPython.utils.io import Tee # io.stdout = Tee(self.cout, channel='stdout') # dbg # io.stderr = Tee(self.cout, channel='stderr') # dbg # Store a few parts of IPython we'll need. self.IP = IP self.user_ns = self.IP.user_ns self.user_global_ns = self.IP.user_global_ns self.input = "" self.output = "" self.is_verbatim = False self.is_doctest = False self.is_suppress = False # Optionally, provide more detailed information to shell. self.directive = None # on the first call to the savefig decorator, we'll import # pyplot as plt so we can make a call to the plt.gcf().savefig self._pyplot_imported = False # Prepopulate the namespace. for line in exec_lines: self.process_input_line(line, store_history=False) def clear_cout(self): self.cout.seek(0) self.cout.truncate(0) def process_input_line(self, line, store_history=True): """process the input, capturing stdout""" stdout = sys.stdout splitter = self.IP.input_splitter try: sys.stdout = self.cout splitter.push(line) more = splitter.push_accepts_more() if not more: source_raw = splitter.source_raw_reset()[1] self.IP.run_cell(source_raw, store_history=store_history) finally: sys.stdout = stdout buflist = self.cout.buflist for i in range(len(buflist)): try: # print(buflist[i]) if not isinstance(buflist[i], unicode): buflist[i] = buflist[i].decode("utf8", "replace") except: pass def process_image(self, decorator): """ # build out an image directive like # .. image:: somefile.png # :width 4in # # from an input like # savefig somefile.png width=4in """ savefig_dir = self.savefig_dir source_dir = self.source_dir saveargs = decorator.split(" ") filename = saveargs[1] # insert relative path to image file in source outfile = os.path.relpath(os.path.join(savefig_dir, filename), source_dir) imagerows = [".. image:: %s" % outfile] for kwarg in saveargs[2:]: arg, val = kwarg.split("=") arg = arg.strip() val = val.strip() imagerows.append(" :%s: %s" % (arg, val)) image_file = os.path.basename(outfile) # only return file name image_directive = "\n".join(imagerows) return image_file, image_directive # Callbacks for each type of token def process_input(self, data, input_prompt, lineno): """ Process data block for INPUT token. """ decorator, input, rest = data image_file = None image_directive = None is_verbatim = decorator == "@verbatim" or self.is_verbatim is_doctest = (decorator is not None and decorator.startswith("@doctest")) or self.is_doctest is_suppress = decorator == "@suppress" or self.is_suppress is_okexcept = decorator == "@okexcept" or self.is_okexcept is_okwarning = decorator == "@okwarning" or self.is_okwarning is_savefig = decorator is not None and decorator.startswith("@savefig") # #>>> required for cython magic to work # def _remove_first_space_if_any(line): # return line[1:] if line.startswith(' ') else line # input_lines = lmap(_remove_first_space_if_any, input.split('\n')) input_lines = input.split("\n") if len(input_lines) > 1: if input_lines[-1] != "": input_lines.append("") # make sure there's a blank line # so splitter buffer gets reset continuation = " %s:" % "".join(["."] * (len(str(lineno)) + 2)) if is_savefig: image_file, image_directive = self.process_image(decorator) ret = [] is_semicolon = False # Hold the execution count, if requested to do so. if is_suppress and self.hold_count: store_history = False else: store_history = True # Note: catch_warnings is not thread safe with warnings.catch_warnings(record=True) as ws: for i, line in enumerate(input_lines): if line.endswith(";"): is_semicolon = True if i == 0: # process the first input line if is_verbatim: self.process_input_line("") self.IP.execution_count += 1 # increment it anyway else: # only submit the line in non-verbatim mode self.process_input_line(line, store_history=store_history) formatted_line = "%s %s" % (input_prompt, line) else: # process a continuation line if not is_verbatim: self.process_input_line(line, store_history=store_history) formatted_line = "%s %s" % (continuation, line) if not is_suppress: ret.append(formatted_line) if not is_suppress and len(rest.strip()) and is_verbatim: # the "rest" is the standard output of the # input, which needs to be added in # verbatim mode ret.append(rest) self.cout.seek(0) output = self.cout.read() if not is_suppress and not is_semicolon: ret.append(output) elif is_semicolon: # get spacing right ret.append("") # context information filename = self.state.document.current_source lineno = self.state.document.current_line # output any exceptions raised during execution to stdout # unless :okexcept: has been specified. if not is_okexcept and "Traceback" in output: s = "\nException in %s at block ending on line %s\n" % (filename, lineno) s += "Specify :okexcept: as an option in the ipython:: block to suppress this message\n" sys.stdout.write("\n\n>>>" + ("-" * 73)) sys.stdout.write(s) sys.stdout.write(output) sys.stdout.write("<<<" + ("-" * 73) + "\n\n") # output any warning raised during execution to stdout # unless :okwarning: has been specified. if not is_okwarning: for w in ws: s = "\nWarning in %s at block ending on line %s\n" % (filename, lineno) s += "Specify :okwarning: as an option in the ipython:: block to suppress this message\n" sys.stdout.write("\n\n>>>" + ("-" * 73)) sys.stdout.write(s) sys.stdout.write("-" * 76 + "\n") s = warnings.formatwarning(w.message, w.category, w.filename, w.lineno, w.line) sys.stdout.write(s) sys.stdout.write("<<<" + ("-" * 73) + "\n") self.cout.truncate(0) return (ret, input_lines, output, is_doctest, decorator, image_file, image_directive) def process_output(self, data, output_prompt, input_lines, output, is_doctest, decorator, image_file): """ Process data block for OUTPUT token. """ TAB = " " * 4 if is_doctest and output is not None: found = output found = found.strip() submitted = data.strip() if self.directive is None: source = "Unavailable" content = "Unavailable" else: source = self.directive.state.document.current_source content = self.directive.content # Add tabs and join into a single string. content = "\n".join([TAB + line for line in content]) # Make sure the output contains the output prompt. ind = found.find(output_prompt) if ind < 0: e = ( "output does not contain output prompt\n\n" "Document source: {0}\n\n" "Raw content: \n{1}\n\n" "Input line(s):\n{TAB}{2}\n\n" "Output line(s):\n{TAB}{3}\n\n" ) e = e.format(source, content, "\n".join(input_lines), repr(found), TAB=TAB) raise RuntimeError(e) found = found[len(output_prompt) :].strip() # Handle the actual doctest comparison. if decorator.strip() == "@doctest": # Standard doctest if found != submitted: e = ( "doctest failure\n\n" "Document source: {0}\n\n" "Raw content: \n{1}\n\n" "On input line(s):\n{TAB}{2}\n\n" "we found output:\n{TAB}{3}\n\n" "instead of the expected:\n{TAB}{4}\n\n" ) e = e.format(source, content, "\n".join(input_lines), repr(found), repr(submitted), TAB=TAB) raise RuntimeError(e) else: self.custom_doctest(decorator, input_lines, found, submitted) def process_comment(self, data): """Process data fPblock for COMMENT token.""" if not self.is_suppress: return [data] def save_image(self, image_file): """ Saves the image file to disk. """ self.ensure_pyplot() command = 'plt.gcf().savefig("%s", bbox_inches="tight", ' "dpi=100)" % image_file # print 'SAVEFIG', command # dbg self.process_input_line("bookmark ipy_thisdir", store_history=False) self.process_input_line("cd -b ipy_savedir", store_history=False) self.process_input_line(command, store_history=False) self.process_input_line("cd -b ipy_thisdir", store_history=False) self.process_input_line("bookmark -d ipy_thisdir", store_history=False) self.clear_cout() def process_block(self, block): """ process block from the block_parser and return a list of processed lines """ ret = [] output = None input_lines = None lineno = self.IP.execution_count input_prompt = self.promptin % lineno output_prompt = self.promptout % lineno image_file = None image_directive = None for token, data in block: if token == COMMENT: out_data = self.process_comment(data) elif token == INPUT: ( out_data, input_lines, output, is_doctest, decorator, image_file, image_directive, ) = self.process_input(data, input_prompt, lineno) elif token == OUTPUT: out_data = self.process_output( data, output_prompt, input_lines, output, is_doctest, decorator, image_file ) if out_data: ret.extend(out_data) # save the image files if image_file is not None: self.save_image(image_file) return ret, image_directive def ensure_pyplot(self): """ Ensures that pyplot has been imported into the embedded IPython shell. Also, makes sure to set the backend appropriately if not set already. """ # We are here if the @figure pseudo decorator was used. Thus, it's # possible that we could be here even if python_mplbackend were set to # `None`. That's also strange and perhaps worthy of raising an # exception, but for now, we just set the backend to 'agg'. if not self._pyplot_imported: if "matplotlib.backends" not in sys.modules: # Then ipython_matplotlib was set to None but there was a # call to the @figure decorator (and ipython_execlines did # not set a backend). # raise Exception("No backend was set, but @figure was used!") import matplotlib matplotlib.use("agg") # Always import pyplot into embedded shell. self.process_input_line("import matplotlib.pyplot as plt", store_history=False) self._pyplot_imported = True def process_pure_python(self, content): """ content is a list of strings. it is unedited directive content This runs it line by line in the InteractiveShell, prepends prompts as needed capturing stderr and stdout, then returns the content as a list as if it were ipython code """ output = [] savefig = False # keep up with this to clear figure multiline = False # to handle line continuation multiline_start = None fmtin = self.promptin ct = 0 for lineno, line in enumerate(content): line_stripped = line.strip() if not len(line): output.append(line) continue # handle decorators if line_stripped.startswith("@"): output.extend([line]) if "savefig" in line: savefig = True # and need to clear figure continue # handle comments if line_stripped.startswith("#"): output.extend([line]) continue # deal with lines checking for multiline continuation = u" %s:" % "".join(["."] * (len(str(ct)) + 2)) if not multiline: modified = u"%s %s" % (fmtin % ct, line_stripped) output.append(modified) ct += 1 try: ast.parse(line_stripped) output.append(u"") except Exception: # on a multiline multiline = True multiline_start = lineno else: # still on a multiline modified = u"%s %s" % (continuation, line) output.append(modified) # if the next line is indented, it should be part of multiline if len(content) > lineno + 1: nextline = content[lineno + 1] if len(nextline) - len(nextline.lstrip()) > 3: continue try: mod = ast.parse("\n".join(content[multiline_start : lineno + 1])) if isinstance(mod.body[0], ast.FunctionDef): # check to see if we have the whole function for element in mod.body[0].body: if isinstance(element, ast.Return): multiline = False else: output.append(u"") multiline = False except Exception: pass if savefig: # clear figure if plotted self.ensure_pyplot() self.process_input_line("plt.clf()", store_history=False) self.clear_cout() savefig = False return output def custom_doctest(self, decorator, input_lines, found, submitted): """ Perform a specialized doctest. """ from .custom_doctests import doctests args = decorator.split() doctest_type = args[1] if doctest_type in doctests: doctests[doctest_type](self, args, input_lines, found, submitted) else: e = "Invalid option to @doctest: {0}".format(doctest_type) raise Exception(e)