def render(self): """supply the :class:`Renderer.Renderer` with the data to render. The data supplied will depend on the ``groupby`` option. return resultblocks """ self.debug("%s: rendering data started for %i items" % (self, len(self.data))) results = ResultBlocks(title="main") # get number of levels required by renderer try: renderer_nlevels = self.renderer.nlevels except AttributeError: renderer_nlevels = 0 data_paths = DataTree.getPaths(self.data) nlevels = len(data_paths) group_level = self.group_level self.debug( "%s: rendering data started. levels=%i, required levels>=%i, group_level=%i, data_paths=%s" % (self, nlevels, renderer_nlevels, group_level, str(data_paths)[:100]) ) if nlevels < renderer_nlevels: # add some dummy levels if levels is not enough d = self.data for x in range(renderer_nlevels - nlevels): d = odict((("all", d),)) results.append(self.renderer(d, path=("all",))) elif group_level < 0 or renderer_nlevels < 0: # no grouping results.append(self.renderer(self.data, path=())) else: # group at level group_level paths = list(itertools.product(*data_paths[: group_level + 1])) for path in paths: work = DataTree.getLeaf(self.data, path) if not work: continue try: results.append(self.renderer(work, path=path)) except: results.append(ResultBlocks(Utils.buildException("rendering"))) if len(results) == 0: self.warn("tracker returned no data.") raise ValueError("tracker returned no data.") self.debug("%s: rendering data finished with %i blocks" % (self.tracker, len(results))) return results
def __call__(self, *args, **kwargs): try: self.parseArguments(*args, **kwargs) except: return ResultBlocks(ResultBlocks(Utils.buildException("parsing"))) self.debug("profile: started: tracker: %s" % (self.tracker)) try: self.collect() except: return ResultBlocks(ResultBlocks(Utils.buildException("collection"))) self.debug("profile: finished: tracker: %s" % (self.tracker)) data_paths = DataTree.getPaths(self.data) self.debug("%s: after collection: %i data_paths: %s" % (self, len(data_paths), str(data_paths))) # transform data try: self.transform() except: return ResultBlocks(ResultBlocks(Utils.buildException("transformation"))) data_paths = DataTree.getPaths(self.data) self.debug("%s: after transformation: %i data_paths: %s" % (self, len(data_paths), str(data_paths))) # remove superfluous levels try: self.prune() except: return ResultBlocks(ResultBlocks(Utils.buildException("pruning"))) data_paths = DataTree.getPaths(self.data) self.debug("%s: after pruning: %i data_paths: %s" % (self, len(data_paths), str(data_paths))) # remove group plots try: self.group() except: return ResultBlocks(ResultBlocks(Utils.buildException("grouping"))) data_paths = DataTree.getPaths(self.data) self.debug("%s: after grouping: %i data_paths: %s" % (self, len(data_paths), str(data_paths))) self.debug("profile: started: renderer: %s" % (self.renderer)) try: result = self.render() except: return ResultBlocks(ResultBlocks(Utils.buildException("rendering"))) self.debug("profile: finished: renderer: %s" % (self.renderer)) return result
######################################################## # create and call dispatcher logging.debug( "report_directive.run: creating dispatcher" ) dispatcher = Dispatcher.Dispatcher( tracker, renderer, transformers ) blocks = dispatcher( **dispatcher_options ) except: logging.warn("report_directive.run: exception caught at %s:%i - see document" % (str(document), lineno) ) blocks = ResultBlocks(ResultBlocks( Utils.buildException( "invocation" ) )) code = None tracker_id = None ######################################################## ## write code output linked_codename = re.sub( "\\\\", "/", os.path.join( rst2srcdir, codename )) if code and basedir != outdir: outfile = open( os.path.join(outdir, codename ), "w" ) for line in code: outfile.write( line ) outfile.close() ########################################################### # collect images ########################################################### map_figure2text = {}
def __call__(self, *args, **kwargs ): #self.debug( "%s: heap at start\n%s" % (self, str(HP.heap()) )) try: self.parseArguments( *args, **kwargs ) except: self.error( "%s: exception in parsing" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "parsing" ) )) # collect no data if tracker is the empty tracker # and go straight to rendering try: if self.tracker.getTracks() == ["empty"]: # is instance does not work because of module mapping # type(Tracker.Empty) == SphinxReport.Tracker.Empty # type(self.tracker) == Tracker.Empty # if isinstance( self.tracker, Tracker.Empty): result =self.renderer() return ResultBlocks( result ) except AttributeError: # for function trackers pass self.debug( "profile: started: tracker: %s" % (self.tracker)) # collecting data try: self.collect() except: self.error( "%s: exception in collection" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "collection" ) )) finally: self.debug( "profile: finished: tracker: %s" % (self.tracker)) if len(self.data) == 0: self.info( "%s: no data - processing complete" % self.tracker ) return None data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after collection: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # self.debug( "%s: heap after collection\n%s" % (self, str(HP.heap()) )) # transform data try: self.transform() except: self.error( "%s: exception in transformation" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "transformation" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after transformation: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # special Renderers - do not proceed # Special renderers if isinstance( self.renderer, Renderer.User): results = ResultBlocks( title="main" ) results.append( self.renderer( self.data, ('') ) ) return results elif isinstance( self.renderer, Renderer.Debug): results = ResultBlocks( title="main" ) results.append( self.renderer( self.data, ('') ) ) return results # self.debug( "%s: heap after transformation\n%s" % (self, str(HP.heap()) )) # restrict try: self.restrict() except: self.error( "%s: exception in restrict" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "restrict" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after restrict: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # exclude try: self.exclude() except: self.error( "%s: exception in exclude" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "exclude" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after exclude: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # remove superfluous levels try: self.prune() except: self.error( "%s: exception in pruning" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "pruning" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after pruning: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # remove group plots try: self.group() except: self.error( "%s: exception in grouping" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "grouping" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after grouping: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) self.debug( "profile: started: renderer: %s" % (self.renderer)) try: result = self.render() except: self.error( "%s: exception in rendering" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "rendering" ) )) finally: self.debug( "profile: finished: renderer: %s" % (self.renderer)) #self.debug( "%s: heap at end\n%s" % (self, str(HP.heap()) )) return result
def render( self ): '''supply the :class:`Renderer.Renderer` with the data to render. The data supplied will depend on the ``groupby`` option. returns a ResultBlocks data structure. ''' self.debug( "%s: rendering data started for %i items" % (self, len(self.data))) # get number of levels required by renderer try: renderer_nlevels = self.renderer.nlevels except AttributeError: # set to -1 to avoid any grouping # important for user renderers that are functions # and have no level attribute. renderer_nlevels = -1 # initiate output structure results = ResultBlocks( title = "") # convert to data series # The data is melted, i.e, # BMW price 10000 # BMW speed 100 # Golf price 5000 # Golf speed 50 dataframe = DataTree.asDataFrame( self.data ) # dataframe.write_csv( "test.csv" ) if dataframe is None: self.warn( "%s: no data after conversion" % self ) raise ValueError( "no data for renderer" ) # special patch: set column names to pruned levels # if there are no column names if len(dataframe.columns) == len(self.pruned): if list(dataframe.columns) == list(range( len(dataframe.columns))): dataframe.columns = [x[1] for x in self.pruned] index = dataframe.index def getIndexLevels( index ): try: # hierarchical index nlevels = len(index.levels) except AttributeError: nlevels = 1 index = [ (x,) for x in index] #raise ValueError('data frame without MultiIndex' ) return nlevels nlevels = getIndexLevels( index ) self.debug( "%s: rendering data started. levels=%i, required levels>=%i, group_level=%s" %\ (self, nlevels, renderer_nlevels, str(self.group_level) ) ) if renderer_nlevels < 0 and self.group_level <= 0: # no grouping for renderers that will accept # a dataframe with any level of indices and no explicit # grouping has been asked for. results.append( self.renderer( dataframe, path = () ) ) else: # user specified group level by default group_level = self.group_level # set group level to maximum allowed by renderer if renderer_nlevels >= 0: group_level = max(nlevels - renderer_nlevels, group_level) # add additional level if necessary if nlevels < group_level: prefix = tuple(["level%i" % x for x in range( group_level - nlevels)]) dataframe.index = pandas.MultiIndex.from_tuples( [ prefix + x for x in dataframe.index ] ) # used to be: group_level + 1 # hierarchical index # numpy.unique converts everything to a string # which is not consistent with selecting later paths = map( tuple, DataTree.unique( [ x[:group_level] for x in dataframe.index.unique() ] )) pathlength = len(paths[0]) - 1 is_hierarchical = isinstance( dataframe.index, pandas.core.index.MultiIndex ) if is_hierarchical: # Note: can only sort hierarchical indices dataframe = dataframe.sortlevel() if dataframe.index.lexsort_depth < pathlength: raise ValueError('could not sort data frame: sort depth=%i < pathlength=%i, dataframe=%s' \ % (dataframe.index.lexsort_depth, pathlength, dataframe)) for path in paths: if path: if len(path) == nlevels: # extract with loc in order to obtain dataframe work = dataframe.loc[[path]] else: # select data frame as cross-section work = dataframe.xs(path, axis=0 ) else: # empty tuple - use full data set work = dataframe # remove columns and rows in work that are all Na work = work.dropna( axis=1, how='all').dropna( axis=0, how='all') if is_hierarchical and renderer_nlevels >= 0: work_levels = getIndexLevels( work.index ) # reduce levels of indices required to that required # for Renderer. This occurs if groupby=none. if work_levels > renderer_nlevels: sep = work_levels - (renderer_nlevels - 1) tuples = [ ( DataTree.path2str( x[:sep] ), ) + x[sep:] \ for x in work.index ] work.index = pandas.MultiIndex.from_tuples( tuples ) try: results.append( self.renderer( work, path = path )) except: self.error( "%s: exception in rendering" % self ) results.append( ResultBlocks( Utils.buildException( "rendering" ) ) ) if len(results) == 0: self.warn("renderer returned no data.") raise ValueError( "renderer returned no data." ) self.debug( "%s: rendering data finished with %i blocks" % (self.tracker, len(results))) return results
def run(arguments, options, lineno, content, state_machine=None, document=None, srcdir=None, builddir=None): """process :report: directive. *srdir* - top level directory of rst documents *builddir* - build directory """ tag = "%s:%i" % (str(document), lineno) logging.debug("report_directive.run: profile: started: rst: %s" % tag) # sort out the paths # reference is used for time-stamping tracker_name = directives.uri(arguments[0]) basedir, fname, basename, ext, outdir, codename, notebookname = Utils.buildPaths(tracker_name) # get the directory of the rst file rstdir, rstfile = os.path.split(document) # state_machine.document.attributes['source']) # root of document tree if srcdir is None: srcdir = setup.srcdir # build directory if builddir is None: builddir = setup.confdir # remove symbolic links srcdir, builddir, rstdir = [os.path.realpath(x) for x in (srcdir, builddir, rstdir)] # there are three directories: # builddir = directory where document is built in (usually _build/html or similar) # rstdir = directory where rst sources are located # srcdir = directory from which the build process is started # path to root relative to rst rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir) # path to root relative to rst rst2builddir = os.path.join(os.path.relpath(builddir, start=rstdir), outdir) # path relative to source (for images) root2builddir = os.path.join(os.path.relpath(builddir, start=srcdir), outdir) logging.debug( "report_directive.run: arguments=%s, options=%s, lineno=%s, content=%s, document=%s" % (str(arguments), str(options), str(lineno), str(content), str(document)) ) logging.debug( "report_directive.run: plotdir=%s, basename=%s, ext=%s, fname=%s, rstdir=%s, srcdir=%s, builddir=%s" % (tracker_name, basename, ext, fname, rstdir, srcdir, builddir) ) logging.debug( "report_directive.run: tracker_name=%s, basedir=%s, rst2src=%s, root2build=%s, outdir=%s, codename=%s" % (tracker_name, basedir, rst2srcdir, rst2builddir, outdir, codename) ) # try to create. If several processes try to create it, # testing with `if` will not work. try: os.makedirs(outdir) except OSError as msg: pass if not os.path.exists(outdir): raise OSError("could not create directory %s: %s" % (outdir, msg)) ######################################################## # collect options # replace placedholders try: options = Utils.updateOptions(options) except ValueError as msg: logging.warn("failure while updating options: %s" % msg) logging.debug("report_directive.run: options=%s" % (str(options),)) transformer_names = [] renderer_name = None # get layout option layout = options.get("layout", "column") option_map = getOptionMap() renderer_options = Utils.selectAndDeleteOptions(options, option_map["render"]) transformer_options = Utils.selectAndDeleteOptions(options, option_map["transform"]) dispatcher_options = Utils.selectAndDeleteOptions(options, option_map["dispatch"]) tracker_options = Utils.selectAndDeleteOptions(options, option_map["tracker"]) display_options = Utils.selectAndDeleteOptions(options, option_map["display"]) logging.debug("report_directive.run: renderer options: %s" % str(renderer_options)) logging.debug("report_directive.run: transformer options: %s" % str(transformer_options)) logging.debug("report_directive.run: dispatcher options: %s" % str(dispatcher_options)) logging.debug("report_directive.run: tracker options: %s" % str(tracker_options)) logging.debug("report_directive.run: display options: %s" % str(display_options)) if "transform" in display_options: transformer_names = display_options["transform"].split(",") del display_options["transform"] if "render" in display_options: renderer_name = display_options["render"] del display_options["render"] ######################################################## # check for missing files if renderer_name != None: options_key = ( str(renderer_options) + str(transformer_options) + str(dispatcher_options) + str(tracker_options) + str(transformer_names) ) options_hash = hashlib.md5(options_key.encode()).hexdigest() template_name = Utils.quote_filename(Config.SEPARATOR.join((tracker_name, renderer_name, options_hash))) filename_text = os.path.join(outdir, "%s.txt" % (template_name)) logging.debug("report_directive.run: options_hash=%s" % options_hash) ########################################################### # check for existing files # update strategy does not use file stamps, but checks # for presence/absence of text element and if all figures # mentioned in the text element are present ########################################################### queries = [re.compile("%s(%s\S+.%s)" % (root2builddir, outdir, suffix)) for suffix in ("png", "pdf")] logging.debug("report_directive.run: checking for changed files.") # check if text element exists if os.path.exists(filename_text): lines = [x[:-1] for x in open(filename_text, "r").readlines()] filenames = [] # check if all figures are present for line in lines: for query in queries: x = query.search(line) if x: filenames.extend(list(x.groups())) logging.debug("report_directive.run: %s: checking for %s" % (tag, str(filenames))) for filename in filenames: if not os.path.exists(filename): logging.info("report_directive.run: %s: redo: %s missing" % (tag, filename)) break else: logging.info("report_directive.run: %s: noredo: all files are present" % tag) ## all is present - save text and return if lines and state_machine: state_machine.insert_input(lines, state_machine.input_lines.source(0)) return [] else: logging.debug("report_directive.run: %s: no check performed: %s missing" % (tag, str(filename_text))) else: template_name = "" filename_text = None ########################################################## # Initialize collectors collectors = [] for collector in list(getPlugins("collect").values()): collectors.append(collector()) ########################################################## ## instantiate tracker, dispatcher, renderer and transformers ## and collect output ########################################################### try: ######################################################## # find the tracker logging.debug("report_directive.run: collecting tracker %s with options %s " % (tracker_name, tracker_options)) code, tracker = Utils.makeTracker(tracker_name, (), tracker_options) if not tracker: logging.error("report_directive.run: no tracker - no output from %s " % str(document)) raise ValueError("tracker `%s` not found" % tracker_name) logging.debug("report_directive.run: collected tracker %s" % tracker_name) tracker_id = Cache.tracker2key(tracker) ######################################################## # determine the transformer logging.debug("report_directive.run: creating transformers") transformers = Utils.getTransformers(transformer_names, transformer_options) ######################################################## # determine the renderer logging.debug("report_directive.run: creating renderer.") if renderer_name == None: logging.error("report_directive.run: no renderer - no output from %s" % str(document)) raise ValueError("the report directive requires a renderer") renderer = Utils.getRenderer(renderer_name, renderer_options) ######################################################## # create and call dispatcher logging.debug("report_directive.run: creating dispatcher") dispatcher = Dispatcher.Dispatcher(tracker, renderer, transformers) # add the tracker options dispatcher_options.update(tracker_options) blocks = dispatcher(**dispatcher_options) if blocks == None: blocks = ResultBlocks( ResultBlocks(Utils.buildWarning("NoData", "tracker %s returned no Data" % str(tracker))) ) code = None tracker_id = None except: logging.warn("report_directive.run: exception caught at %s:%i - see document" % (str(document), lineno)) blocks = ResultBlocks(ResultBlocks(Utils.buildException("invocation"))) code = None tracker_id = None logging.debug("report_directive.run: profile: started: collecting: %s" % tag) ######################################################## ## write code output linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename)) if code and basedir != outdir: with open(os.path.join(outdir, codename), "w") as outfile: for line in code: outfile.write(line) ######################################################## ## write notebook snippet linked_notebookname = re.sub("\\\\", "/", os.path.join(rst2srcdir, notebookname)) if basedir != outdir: with open(os.path.join(outdir, notebookname), "w") as outfile: Utils.writeNoteBookEntry( outfile, renderer=renderer_name, tracker=tracker_name, transformers=transformer_names, options=renderer_options.items() + tracker_options.items() + transformer_options.items(), ) ########################################################### # collect images ########################################################### map_figure2text = {} try: for collector in collectors: map_figure2text.update( collector.collect( blocks, template_name, outdir, rstdir, builddir, srcdir, content, display_options, tracker_id, links={"code_url": linked_codename, "notebook_url": linked_notebookname}, ) ) except: logging.warn( "report_directive.run: exception caught while collecting with %s at %s:%i - see document" % (collector, str(document), lineno) ) blocks = ResultBlocks(ResultBlocks(Utils.buildException("collection"))) code = None tracker_id = None ########################################################### # replace place holders or add text ########################################################### ## add default for text-only output map_figure2text["default-prefix"] = TEMPLATE_TEXT % locals() map_figure2text["default-suffix"] = "" blocks.updatePlaceholders(map_figure2text) ########################################################### ## render the output taking into account the layout lines = Utils.layoutBlocks(blocks, layout) ########################################################### # add caption lines.extend(["::", ""]) if content: lines.extend([" %s" % row.strip() for row in content]) lines.append("") lines.append("") # output rst text for this renderer if filename_text: outfile = open(filename_text, "w") outfile.write("\n".join(lines)) outfile.close() if SPHINXREPORT_DEBUG: for x, l in enumerate(lines): print("%5i %s" % (x, l)) if len(lines) and state_machine: state_machine.insert_input(lines, state_machine.input_lines.source(0)) logging.debug("report_directive.run: profile: finished: collecting: %s" % tag) logging.debug("report_directive.run: profile: finished: rst: %s:%i" % (str(document), lineno)) return []