def run(arguments, options, lineno, content, state_machine=None, document=None, srcdir=None, builddir=None): """process :report: directive. *srdir* - top level directory of rst documents *builddir* - build directory """ tag = "%s:%i" % (str(document), lineno) logging.debug("report_directive.run: profile: started: rst: %s" % tag) # sort out the paths # reference is used for time-stamping tracker_name = directives.uri(arguments[0]) basedir, fname, basename, ext, outdir, codename, notebookname = Utils.buildPaths(tracker_name) # get the directory of the rst file rstdir, rstfile = os.path.split(document) # state_machine.document.attributes['source']) # root of document tree if srcdir is None: srcdir = setup.srcdir # build directory if builddir is None: builddir = setup.confdir # remove symbolic links srcdir, builddir, rstdir = [os.path.realpath(x) for x in (srcdir, builddir, rstdir)] # there are three directories: # builddir = directory where document is built in (usually _build/html or similar) # rstdir = directory where rst sources are located # srcdir = directory from which the build process is started # path to root relative to rst rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir) # path to root relative to rst rst2builddir = os.path.join(os.path.relpath(builddir, start=rstdir), outdir) # path relative to source (for images) root2builddir = os.path.join(os.path.relpath(builddir, start=srcdir), outdir) logging.debug( "report_directive.run: arguments=%s, options=%s, lineno=%s, content=%s, document=%s" % (str(arguments), str(options), str(lineno), str(content), str(document)) ) logging.debug( "report_directive.run: plotdir=%s, basename=%s, ext=%s, fname=%s, rstdir=%s, srcdir=%s, builddir=%s" % (tracker_name, basename, ext, fname, rstdir, srcdir, builddir) ) logging.debug( "report_directive.run: tracker_name=%s, basedir=%s, rst2src=%s, root2build=%s, outdir=%s, codename=%s" % (tracker_name, basedir, rst2srcdir, rst2builddir, outdir, codename) ) # try to create. If several processes try to create it, # testing with `if` will not work. try: os.makedirs(outdir) except OSError as msg: pass if not os.path.exists(outdir): raise OSError("could not create directory %s: %s" % (outdir, msg)) ######################################################## # collect options # replace placedholders try: options = Utils.updateOptions(options) except ValueError as msg: logging.warn("failure while updating options: %s" % msg) logging.debug("report_directive.run: options=%s" % (str(options),)) transformer_names = [] renderer_name = None # get layout option layout = options.get("layout", "column") option_map = getOptionMap() renderer_options = Utils.selectAndDeleteOptions(options, option_map["render"]) transformer_options = Utils.selectAndDeleteOptions(options, option_map["transform"]) dispatcher_options = Utils.selectAndDeleteOptions(options, option_map["dispatch"]) tracker_options = Utils.selectAndDeleteOptions(options, option_map["tracker"]) display_options = Utils.selectAndDeleteOptions(options, option_map["display"]) logging.debug("report_directive.run: renderer options: %s" % str(renderer_options)) logging.debug("report_directive.run: transformer options: %s" % str(transformer_options)) logging.debug("report_directive.run: dispatcher options: %s" % str(dispatcher_options)) logging.debug("report_directive.run: tracker options: %s" % str(tracker_options)) logging.debug("report_directive.run: display options: %s" % str(display_options)) if "transform" in display_options: transformer_names = display_options["transform"].split(",") del display_options["transform"] if "render" in display_options: renderer_name = display_options["render"] del display_options["render"] ######################################################## # check for missing files if renderer_name != None: options_key = ( str(renderer_options) + str(transformer_options) + str(dispatcher_options) + str(tracker_options) + str(transformer_names) ) options_hash = hashlib.md5(options_key.encode()).hexdigest() template_name = Utils.quote_filename(Config.SEPARATOR.join((tracker_name, renderer_name, options_hash))) filename_text = os.path.join(outdir, "%s.txt" % (template_name)) logging.debug("report_directive.run: options_hash=%s" % options_hash) ########################################################### # check for existing files # update strategy does not use file stamps, but checks # for presence/absence of text element and if all figures # mentioned in the text element are present ########################################################### queries = [re.compile("%s(%s\S+.%s)" % (root2builddir, outdir, suffix)) for suffix in ("png", "pdf")] logging.debug("report_directive.run: checking for changed files.") # check if text element exists if os.path.exists(filename_text): lines = [x[:-1] for x in open(filename_text, "r").readlines()] filenames = [] # check if all figures are present for line in lines: for query in queries: x = query.search(line) if x: filenames.extend(list(x.groups())) logging.debug("report_directive.run: %s: checking for %s" % (tag, str(filenames))) for filename in filenames: if not os.path.exists(filename): logging.info("report_directive.run: %s: redo: %s missing" % (tag, filename)) break else: logging.info("report_directive.run: %s: noredo: all files are present" % tag) ## all is present - save text and return if lines and state_machine: state_machine.insert_input(lines, state_machine.input_lines.source(0)) return [] else: logging.debug("report_directive.run: %s: no check performed: %s missing" % (tag, str(filename_text))) else: template_name = "" filename_text = None ########################################################## # Initialize collectors collectors = [] for collector in list(getPlugins("collect").values()): collectors.append(collector()) ########################################################## ## instantiate tracker, dispatcher, renderer and transformers ## and collect output ########################################################### try: ######################################################## # find the tracker logging.debug("report_directive.run: collecting tracker %s with options %s " % (tracker_name, tracker_options)) code, tracker = Utils.makeTracker(tracker_name, (), tracker_options) if not tracker: logging.error("report_directive.run: no tracker - no output from %s " % str(document)) raise ValueError("tracker `%s` not found" % tracker_name) logging.debug("report_directive.run: collected tracker %s" % tracker_name) tracker_id = Cache.tracker2key(tracker) ######################################################## # determine the transformer logging.debug("report_directive.run: creating transformers") transformers = Utils.getTransformers(transformer_names, transformer_options) ######################################################## # determine the renderer logging.debug("report_directive.run: creating renderer.") if renderer_name == None: logging.error("report_directive.run: no renderer - no output from %s" % str(document)) raise ValueError("the report directive requires a renderer") renderer = Utils.getRenderer(renderer_name, renderer_options) ######################################################## # create and call dispatcher logging.debug("report_directive.run: creating dispatcher") dispatcher = Dispatcher.Dispatcher(tracker, renderer, transformers) # add the tracker options dispatcher_options.update(tracker_options) blocks = dispatcher(**dispatcher_options) if blocks == None: blocks = ResultBlocks( ResultBlocks(Utils.buildWarning("NoData", "tracker %s returned no Data" % str(tracker))) ) code = None tracker_id = None except: logging.warn("report_directive.run: exception caught at %s:%i - see document" % (str(document), lineno)) blocks = ResultBlocks(ResultBlocks(Utils.buildException("invocation"))) code = None tracker_id = None logging.debug("report_directive.run: profile: started: collecting: %s" % tag) ######################################################## ## write code output linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename)) if code and basedir != outdir: with open(os.path.join(outdir, codename), "w") as outfile: for line in code: outfile.write(line) ######################################################## ## write notebook snippet linked_notebookname = re.sub("\\\\", "/", os.path.join(rst2srcdir, notebookname)) if basedir != outdir: with open(os.path.join(outdir, notebookname), "w") as outfile: Utils.writeNoteBookEntry( outfile, renderer=renderer_name, tracker=tracker_name, transformers=transformer_names, options=renderer_options.items() + tracker_options.items() + transformer_options.items(), ) ########################################################### # collect images ########################################################### map_figure2text = {} try: for collector in collectors: map_figure2text.update( collector.collect( blocks, template_name, outdir, rstdir, builddir, srcdir, content, display_options, tracker_id, links={"code_url": linked_codename, "notebook_url": linked_notebookname}, ) ) except: logging.warn( "report_directive.run: exception caught while collecting with %s at %s:%i - see document" % (collector, str(document), lineno) ) blocks = ResultBlocks(ResultBlocks(Utils.buildException("collection"))) code = None tracker_id = None ########################################################### # replace place holders or add text ########################################################### ## add default for text-only output map_figure2text["default-prefix"] = TEMPLATE_TEXT % locals() map_figure2text["default-suffix"] = "" blocks.updatePlaceholders(map_figure2text) ########################################################### ## render the output taking into account the layout lines = Utils.layoutBlocks(blocks, layout) ########################################################### # add caption lines.extend(["::", ""]) if content: lines.extend([" %s" % row.strip() for row in content]) lines.append("") lines.append("") # output rst text for this renderer if filename_text: outfile = open(filename_text, "w") outfile.write("\n".join(lines)) outfile.close() if SPHINXREPORT_DEBUG: for x, l in enumerate(lines): print("%5i %s" % (x, l)) if len(lines) and state_machine: state_machine.insert_input(lines, state_machine.input_lines.source(0)) logging.debug("report_directive.run: profile: finished: collecting: %s" % tag) logging.debug("report_directive.run: profile: finished: rst: %s:%i" % (str(document), lineno)) return []
def main( argv = None, **kwargs ): '''main function for test.py. Long-form of command line arguments can also be supplied as kwargs. If argv is not None, command line parsing will be performed. ''' parser = optparse.OptionParser( version = "%prog version: $Id$", usage = globals()["__doc__"] ) parser.add_option( "-t", "--tracker", dest="tracker", type="string", help="tracker to use [default=%default]" ) parser.add_option( "-p", "--page", dest="page", type="string", help="render an rst page [default=%default]" ) parser.add_option( "-a", "--tracks", dest="tracks", type="string", help="tracks to use [default=%default]" ) parser.add_option( "-m", "--transformer", dest="transformers", type="string", action="append", help="add transformation [default=%default]" ) parser.add_option( "-s", "--slices", dest="slices", type="string", help="slices to use [default=%default]" ) parser.add_option( "-r", "--renderer", dest="renderer", type="string", help="renderer to use [default=%default]" ) parser.add_option( "-w", "--path", "--trackerdir", dest="trackerdir", type="string", help="path to trackers [default=%default]" ) parser.add_option( "-f", "--force", dest="force", action="store_true", help="force recomputation of data by deleting cached results [default=%default]" ) parser.add_option( "-o", "--option", dest="options", type="string", action="append", help="renderer options - supply as key=value pairs (without spaces). [default=%default]" ) parser.add_option( "-l", "--language", dest="language", type="choice", choices = ("rst", "notebook"), help="output language for snippet. Use ``rst`` to create a snippet to paste " "into a sphinxreport document. Use ``notebook`` to create a snippet to paste " "into an ipython notebook [default=%default]" ) parser.add_option( "--no-print", dest="do_print", action="store_false", help = "do not print an rst text element to create the displayed plots [default=%default]." ) parser.add_option( "--no-show", dest="do_show", action="store_false", help = "do not show a plot [default=%default]." ) parser.add_option( "-i", "--start-interpreter", dest="start_interpreter", action="store_true", help = "do not render, but start python interpreter [default=%default]." ) parser.add_option( "-I", "--ii", "--start-ipython", dest="start_ipython", action="store_true", help = "do not render, start ipython interpreter [default=%default]." ) parser.add_option( "--workdir", dest="workdir", type="string", help="working directory - change to this directory before executing " " [default=%default]" ) parser.add_option( "--hardcopy", dest="hardcopy", type="string", help = "output images of plots. The parameter should contain one or more %s " " The suffix determines the type of plot. " " [default=%default]." ) parser.set_defaults( loglevel = 1, tracker=None, transformers = [], tracks=None, slices=None, options = [], renderer = "table", do_show = True, do_print = True, force = False, trackerdir = TRACKERDIR, label = "GenericLabel", caption = "add caption here", start_interpreter = False, start_ipython = False, language = "rst", workdir = None, dpi = 100 ) if argv is None and len(kwargs) == 0: argv = sys.argv if argv: (options, args) = parser.parse_args( argv ) if len(args) == 2: options.tracker, options.renderer = args else: (options, args) = parser.parse_args( [] ) ###################################################### # set keyword arguments as options for keyword, value in kwargs.items(): if hasattr( options, keyword ): setattr( options, keyword, value ) del kwargs[keyword] if options.workdir is not None: savedir = os.getcwd() os.chdir( options.workdir ) else: savedir = None ###################################################### # configure options options.trackerdir = os.path.abspath( os.path.expanduser( options.trackerdir ) ) if not os.path.exists( options.trackerdir ): raise IOError("directory %s does not exist" % options.trackerdir ) sys.path.insert( 0, options.trackerdir ) ###################################################### # test plugins for x in options.options: if "=" in x: data = x.split("=") key,val = [ y.strip() for y in (data[0], "=".join(data[1:])) ] else: key, val = x.strip(), None kwargs[key] = val if options.tracks: kwargs["tracks"] = options.tracks if options.slices: kwargs["slices"] = options.slices kwargs = Utils.updateOptions( kwargs ) option_map = getOptionMap() renderer_options = Utils.selectAndDeleteOptions( kwargs, option_map["render"]) transformer_options = Utils.selectAndDeleteOptions( kwargs, option_map["transform"]) display_options = Utils.selectAndDeleteOptions( kwargs, option_map["display"]) ###################################################### # decide whether to render or not if options.renderer == "none" or options.start_interpreter or \ options.start_ipython or options.language == "notebook": renderer = None else: renderer = Utils.getRenderer( options.renderer, renderer_options ) transformers = Utils.getTransformers( options.transformers, transformer_options ) exclude = set( ("Tracker", "TrackerSQL", "returnLabeledData", "returnMultipleColumnData", "returnMultipleColumns", "returnSingleColumn", "returnSingleColumnData", "SQLError", "MultipleColumns", "MultipleColumnData", "LabeledData", "DataSimple", "Data" ) ) ###################################################### ## build from tracker if options.tracker: trackers = [] for filename in glob.glob( os.path.join( options.trackerdir, "*.py" )): modulename = os.path.basename( filename ) trackers.extend( [ x for x in getTrackers( modulename ) if x[0] not in exclude ] ) if "." in options.tracker: parts = options.tracker.split(".") tracker_modulename = ".".join( parts[:-1] ) tracker_name = parts[-1] else: tracker_modulename = None tracker_name = options.tracker for name, tracker, modulename, is_derived in trackers: if name == tracker_name: if tracker_modulename is not None: if modulename == tracker_modulename: break else: break else: available_trackers = set( [ x[0] for x in trackers if x[3] ] ) print("unknown tracker '%s': possible trackers are\n %s" % (options.tracker, "\n ".join( sorted(available_trackers)) )) print("(the list above does not contain functions).") sys.exit(1) ## remove everything related to that tracker for a clean slate if options.force: removed = SphinxReport.clean.removeTracker( name ) print("removed all data for tracker %s: %i files" % (name, len(removed))) # instantiate functors if is_derived: t = tracker( **kwargs ) # but not functions else: t = tracker dispatcher = Dispatcher( t, renderer, transformers ) if renderer == None: dispatcher.parseArguments( **kwargs ) result = dispatcher.collect() result = dispatcher.transform() options.do_print = options.language == "notebook" options.do_show = False options.hardcopy = False else: # needs to be resolved between renderer and dispatcher options result = dispatcher( **kwargs ) if options.do_print: sys.stdout.write(".. Template start\n\n") if options.language == "rst": writeRST( sys.stdout, options, kwargs, renderer_options, transformer_options, display_options, modulename, name) elif options.language == "notebook": writeNotebook( sys.stdout, options, kwargs, renderer_options, transformer_options, display_options, modulename, name) sys.stdout.write ("\n.. Template end\n") if result and renderer != None: for r in result: if r.title: print ("") print ("title: %s" % r.title) print ("") for s in r: print(str(s)) if options.hardcopy: fig_managers = _pylab_helpers.Gcf.get_all_fig_managers() # create all the images for figman in fig_managers: # create all images figid = figman.num outfile = re.sub( "%s", str(figid), options.hardcopy) figman.canvas.figure.savefig( outfile, dpi=options.dpi ) if result and options.do_show: if options.renderer.startswith("r-"): for rr in result: for r in rr: if hasattr(r, 'rggplot'): from rpy2.robjects import r as R import rpy2.rinterface try: R.plot( r.rggplot) except rpy2.rinterface.RRuntimeError, msg: if re.search("object.*not found", str(msg)): print '%s: available columns in dataframe=%s' % \ ( msg, R('''colnames(rframe)''')) print("press Ctrl-c to stop") while 1: pass elif len(_pylab_helpers.Gcf.get_all_fig_managers()) > 0: plt.show() else: for rr in result: for r in rr: if hasattr( r, 'xls' ): tmpfile, outpath = tempfile.mkstemp( dir ='.', suffix = '.xlsx' ) os.close(tmpfile) print ('saving xlsx to %s' % outpath ) r.xls.save( outpath )