def render( self, work, path ): # initiate output structure results = ResultBlocks( title = path ) # iterate over all items at leaf for key in work: t = type(work[key]) try: l = "%i" % len(work[key]) except AttributeError: l = "na" except TypeError: l = "na" # add a result block. data = str(work[key]) if len(data) > 30: data=data[:30] + "..." results.append( ResultBlock( "path= %s, type= %s, len= %s, data= %s" % \ ( path2str(path + (key,)), t, l, data), title = "") ) return results
def render(self, dataframe, path ): if len(dataframe.columns) < 2: raise ValueError( "requiring two coordinates, only got %s" % str(dataframe.columns)) plts, legend = [], [] blocks = ResultBlocks() for xcolumn, ycolumn in itertools.combinations( dataframe.columns, 2 ): # remove missing data points xvalues, yvalues = Stats.filterMissing( (dataframe[xcolumn], dataframe[ycolumn]) ) # remove columns with all NaN if len(xvalues) == 0 or len(yvalues) == 0: continue # apply log transformation on data not on plot if self.logscale: if "x" in self.logscale: xvalues = R.log10(xvalues) if "y" in self.logscale: yvalues = R.log10(yvalues) self.startPlot() # wrap, as pandas series can not # passed through rpy2. R.smoothScatter( numpy.array( xvalues, dtype=numpy.float), numpy.array( yvalues, dtype=numpy.float), xlab=xcolumn, ylab=ycolumn, nbin = self.nbins ) blocks.extend( self.endPlot( dataframe, path ) ) return blocks
def __call__(self, data, path ): '''iterate over leaves/branches in data structure. This method will call the :meth:`render` method for each leaf/branch at level :attr:`nlevels`. ''' if self.nlevels == None: raise NotImplementedError("incomplete implementation of %s" % str(self)) result = ResultBlocks( title = path2str(path) ) labels = DataTree.getPaths( data ) if len(labels) < self.nlevels: self.warn( "at %s: expected at least %i levels - got %i: %s" %\ (str(path), self.nlevels, len(labels), str(labels)) ) result.append( EmptyResultBlock( title = path2str(path) ) ) return result paths = list(itertools.product( *labels[:-self.nlevels] )) for p in paths: work = DataTree.getLeaf( data, p ) if not work: continue try: result.extend( self.render( work, path + p ) ) except: self.warn("exeception raised in rendering for path: %s" % str(path+p)) raise return result
def __call__(self, track, slice=None): # note there are spaces behind the %(image)s directive to accomodate # for path substitution block = ''' .. figure:: %(image)s :height: 300 ''' blocks = ResultBlocks() tracks = sorted([x.asFile() for x in TRACKS]) for track in tracks: files = glob.glob( os.path.join(EXPORTDIR, "fastqc", "%s*_fastqc" % track)) for x, fn in enumerate(sorted(files)): y = x + 1 image = os.path.abspath( os.path.join(fn, "Images", "%s.png" % slice)) if not os.path.exists(image): continue blocks.append(ResultBlock(text=block % locals(), title=os.path.basename(fn))) return odict((("rst", "\n".join(Utils.layoutBlocks(blocks, layout="columns-2"))),))
def render( self, work, path ): """render the data. """ results = ResultBlocks( title = path ) matrix, rows, columns = self.buildMatrix( work ) title = path2str(path) if len(rows) == 0: return ResultBlocks( ResultBlock( "", title = title) ) # do not output large matrices as rst files if not self.force and (len(rows) > self.max_rows or len(columns) > self.max_cols): return self.asFile( [ [ self.toString(x) for x in r ] for r in matrix ], rows, columns, title ) lines = [] lines.append( ".. csv-table:: %s" % title ) lines.append( ' :header: "track","%s" ' % '","'.join( columns ) ) lines.append( '') for x in range(len(rows)): lines.append( ' "%s","%s"' % (rows[x], '","'.join( [ self.toString(x) for x in matrix[x,:] ] ) ) ) lines.append( "") if not path: subtitle = "" else: subtitle = path2str(path) results.append( ResultBlock( "\n".join(lines), title = subtitle ) ) return results
def __call__(self, track, slice=None): # note there are spaces behind the %(image)s directive to accomodate # for path substitution block = ''' .. figure:: %(image)s :height: 300 ''' blocks = ResultBlocks() tracks = sorted([x.asFile() for x in TRACKS]) for track in tracks: files = glob.glob( os.path.join(EXPORTDIR, "fastqc", "%s*_fastqc" % track)) for x, fn in enumerate(sorted(files)): y = x + 1 image = os.path.abspath( os.path.join(fn, "Images", "%s.png" % slice)) if not os.path.exists(image): continue blocks.append( ResultBlock(text=block % locals(), title=os.path.basename(fn))) return odict( (("rst", "\n".join(Utils.layoutBlocks(blocks, layout="columns-2"))), ))
def __call__(self, track, slice = None ): edir = EXPORTDIR method = "utr_extension" blocks = ResultBlocks() filepath = "%(edir)s/%(method)s/%(track)s.readextension_%(region)s_%(direction)s.%(slice)s.png" block = \ ''' .. figure:: %(filename)s :height: 300 ''' # append spaces for file extension block = "\n".join( [ x + " " * 200 for x in block.split("\n") ] ) for region, direction in itertools.product( ("downstream", "upstream"), ("sense", "antisense", "anysense" )): filename = filepath % locals() if os.path.exists( filename ): blocks.append( ResultBlock( text = block % locals(), title = "%(track)s %(region)s %(direction)s" % locals() ) ) # else: # blocks.append( ResultBlock( "", # title = "%(track)s %(region)s %(direction)s" % locals() ) ) return odict( (("rst", "\n".join(Utils.layoutBlocks( blocks, layout="columns-3" ))),))
def render( self, data, path ): # initiate output structure results = ResultBlocks( title = path ) try: results.append( ResultBlock(json.dumps(data, indent=4), title='')) except TypeError: results.append( ResultBlock(str(data), title='')) return results
def __call__(self, track, slice = None ): blocks = ResultBlocks() block = ''' .. figure:: %(image)s :height: 300 ''' for image in glob.glob( os.path.join( IMAGEDIR, "*.png" )): blocks.append( ResultBlock( text = block % locals(), title = "image" ) ) return odict( (("rst", "\n".join( Utils.layoutBlocks( blocks, layout = "columns-2"))),))
def render(self): """supply the :class:`Renderer.Renderer` with the data to render. The data supplied will depend on the ``groupby`` option. return resultblocks """ self.debug("%s: rendering data started for %i items" % (self, len(self.data))) results = ResultBlocks(title="main") # get number of levels required by renderer try: renderer_nlevels = self.renderer.nlevels except AttributeError: renderer_nlevels = 0 data_paths = DataTree.getPaths(self.data) nlevels = len(data_paths) group_level = self.group_level self.debug( "%s: rendering data started. levels=%i, required levels>=%i, group_level=%i, data_paths=%s" % (self, nlevels, renderer_nlevels, group_level, str(data_paths)[:100]) ) if nlevels < renderer_nlevels: # add some dummy levels if levels is not enough d = self.data for x in range(renderer_nlevels - nlevels): d = odict((("all", d),)) results.append(self.renderer(d, path=("all",))) elif group_level < 0 or renderer_nlevels < 0: # no grouping results.append(self.renderer(self.data, path=())) else: # group at level group_level paths = list(itertools.product(*data_paths[: group_level + 1])) for path in paths: work = DataTree.getLeaf(self.data, path) if not work: continue try: results.append(self.renderer(work, path=path)) except: results.append(ResultBlocks(Utils.buildException("rendering"))) if len(results) == 0: self.warn("tracker returned no data.") raise ValueError("tracker returned no data.") self.debug("%s: rendering data finished with %i blocks" % (self.tracker, len(results))) return results
def render( self, work, path ): # initiate output structure results = ResultBlocks( title = path2str(path) ) labels = DataTree.getPaths( work ) # iterate over all items at leaf for path, branch in DataTree.getNodes( work, len(labels) - 2 ): for key in Utils.TrackerKeywords: if key in branch: # add a result block results.append( ResultBlock( branch[key], title = path2str(path) ) ) return results
def __call__(self, dataframe, path ): '''iterate over leaves/branches in data structure. This method will call the :meth:`render` method for each leaf/branch at level :attr:`nlevels`. ''' if self.nlevels == None: raise NotImplementedError("incomplete implementation of %s" % str(self)) try: labels = dataframe.index.levels paths = dataframe.index.unique() except AttributeError: labels = ['dummy1'] paths = ['dummy1'] result = ResultBlocks() #print len(labels), self.nlevels #print 'dataframe=', dataframe if self.nlevels != -1 and len(labels) != self.nlevels: raise ValueError( "at path %s: expected %i levels - got %i: %s" %\ (str(path), self.nlevels, len(labels), str(labels)) ) #result.append( EmptyResultBlock( title = path2str(path) ) ) #return result if not self.split_at: # print without splitting result.extend( self.render( dataframe, path ) ) else: # split dataframe at first index first_level_labels = dataframe.index.get_level_values(0).unique() if len(first_level_labels) < self.split_at: result.extend( self.render( dataframe, path ) ) else: # select tracks to always add to split # pick always tracks if self.split_always: always = [ x for x, y in itertools.product( first_level_labels, self.split_always) \ if re.search( y, x ) ] else: always = [] for z, x in enumerate(range( 0, len(first_level_labels), self.split_at)) : select = list(DataTree.unique( always + list(first_level_labels[x:x+self.split_at]) )) if len(dataframe.index.names) == 1: # if only one level, use loc to obtain dataframe # index is duplicated, so ignore second level work = pandas.concat( [dataframe.loc[[s]] for s in select], keys = select ) work.reset_index( range( 1, len(work.index.names)), drop=True, inplace=True ) else: work = pandas.concat( [dataframe.xs(s, axis=0) for s in select], keys = select ) # reconcile index names work.index.names = dataframe.index.names result.extend( self.render( work, path + (z, ) ) ) return result
def render( self, work, path ): # initiate output structure results = ResultBlocks( title = path ) # iterate over all items at leaf #for key in work: if "text" in work: # add a result block results.append( ResultBlock( work["text"], title = "" ) ) elif "rst" in work: # add a result block results.append( ResultBlock( work["rst"], title = "" ) ) return results
def __call__(self, dataframe, path): # modify table (adding/removing columns) according to user options # matrix, row_headers, col_headers = self.modifyTable( matrix, row_headers, col_headers ) dataframe = self.modifyTable( dataframe ) title = path2str(path) results = ResultBlocks() row_headers = dataframe.index col_headers = dataframe.columns # do not output large matrices as rst files if self.separate or (not self.force and (len(row_headers) > self.max_rows or len(col_headers) > self.max_cols)): if self.large == "xls": results.append( self.asSpreadSheet( dataframe, row_headers, col_headers, title ) ) else: results.append( self.asFile( dataframe, row_headers, col_headers, title ) ) if self.preview: row_headers = row_headers[:self.max_rows] col_headers = col_headers[:self.max_cols] matrix = [ x[:self.max_cols] for x in matrix[:self.max_rows] ] else: return results out = StringIO.StringIO() dataframe.to_csv( out ) lines = [] lines.append( ".. csv-table:: %s" % title ) lines.append( " :class: sortable" ) if self.add_rowindex: lines.append( ' :header: "row", "", "%s" ' % '","'.join( map(str, col_headers) ) ) lines.append( '' ) x = 0 for header, line in zip( row_headers, matrix ): x += 1 lines.append( ' %i,"%s","%s"' % (x, str(header), '","'.join( map(str, line) ) ) ) else: l = out.getvalue().split("\n") lines.append( ' :header: %s' % l[0] ) lines.append( '' ) lines.extend( [' %s' % x for x in l[1:] ] ) lines.append( "") results.append( ResultBlock( "\n".join(lines), title = title) ) return results
def render( self, work, path ): """render the data. """ results = ResultBlocks( title = path ) matrix, rows, columns = self.buildMatrix( work ) title = path2str(path) if len(rows) == 0: return ResultBlocks( ResultBlock( "", title = title) ) # do not output large matrices as rst files # separate and force need to be mixed in. if self.separate or (not self.force and (len(rows) > self.max_rows or len(columns) > self.max_cols)): return ResultBlocks( self.asFile( pandas.DataFrame( matrix, index = rows, columns = columns), rows, columns, title ), title = path ) lines = [] lines.append( ".. csv-table:: %s" % title ) lines.append( " :class: sortable" ) lines.append( ' :header: "track","%s" ' % '","'.join( columns ) ) lines.append( '') for x in range(len(rows)): lines.append( ' "%s","%s"' % (rows[x], '","'.join( [ self.toString(x) for x in matrix[x,:] ] ) ) ) lines.append( "") if path is None: subtitle = "" else: subtitle = path2str(path) results.append( ResultBlock( "\n".join(lines), title = subtitle ) ) return results
######################################################## # create and call dispatcher logging.debug( "report_directive.run: creating dispatcher" ) dispatcher = Dispatcher.Dispatcher( tracker, renderer, transformers ) blocks = dispatcher( **dispatcher_options ) except: logging.warn("report_directive.run: exception caught at %s:%i - see document" % (str(document), lineno) ) blocks = ResultBlocks(ResultBlocks( Utils.buildException( "invocation" ) )) code = None tracker_id = None ######################################################## ## write code output linked_codename = re.sub( "\\\\", "/", os.path.join( rst2srcdir, codename )) if code and basedir != outdir: outfile = open( os.path.join(outdir, codename ), "w" ) for line in code: outfile.write( line ) outfile.close() ########################################################### # collect images ########################################################### map_figure2text = {}
def layoutBlocks( blocks, layout = "column"): """layout blocks of rst text. layout can be one of "column", "row", or "grid". The layout uses an rst table to arrange elements. """ lines = [] if len(blocks) == 0: return lines # flatten blocks x = ResultBlocks() for b in blocks: if b.title: b.updateTitle( b.title, "prefix" ) try: x.extend( b ) except TypeError: x.append( b ) blocks = x if layout == "column": for block in blocks: if block.title: lines.extend( block.title.split("\n") ) lines.append( "" ) else: logging.warn( "report_directive.layoutBlocks: missing title" ) lines.extend(block.text.split("\n")) lines.extend( [ "", ] ) return lines elif layout in ("row", "grid"): if layout == "row": ncols = len(blocks) elif layout == "grid": ncols = int(math.ceil(math.sqrt( len(blocks) ))) elif layout.startswith("column"): ncols = min( len(blocks), int(layout.split("-")[1])) # TODO: think about appropriate fix for empty data if ncols == 0: ncols = 1 return lines else: raise ValueError( "unknown layout %s " % layout ) # compute column widths widths = [ x.getWidth() for x in blocks ] text_heights = [ x.getTextHeight() for x in blocks ] title_heights = [ x.getTitleHeight() for x in blocks ] columnwidths = [] for x in range(ncols): columnwidths.append( max( [widths[y] for y in range( x, len(blocks), ncols ) ] ) ) separator = "+%s+" % "+".join( ["-" * x for x in columnwidths ] ) ## add empty blocks if len(blocks) % ncols: blocks.extend( [ ResultBlock( "", "" )] * (ncols - len(blocks) % ncols) ) for nblock in range(0, len(blocks), ncols ): ## add text lines.append( separator ) max_height = max( text_heights[nblock:nblock+ncols] ) new_blocks = ResultBlocks() for xx in range(nblock, min(nblock+ncols,len(blocks))): txt, col = blocks[xx].text.split("\n"), xx % ncols max_width = columnwidths[col] # add missig lines txt.extend( [""] * (max_height - len(txt)) ) # extend lines txt = [ x + " " * (max_width - len(x)) for x in txt ] new_blocks.append( txt ) for l in zip( *new_blocks ): lines.append( "|%s|" % "|".join( l ) ) ## add subtitles max_height = max( title_heights[nblock:nblock+ncols] ) if max_height > 0: new_blocks = ResultBlocks() lines.append( separator ) for xx in range(nblock, min(nblock+ncols,len(blocks))): txt, col = blocks[xx].title.split("\n"), xx % ncols max_width = columnwidths[col] # add missig lines txt.extend( [""] * (max_height - len(txt) ) ) # extend lines txt = [ x + " " * (max_width - len(x)) for x in txt ] new_blocks.append( txt ) for l in zip( *new_blocks ): lines.append( "|%s|" % "|".join( l ) ) lines.append( separator ) lines.append( "" ) return lines
def run(arguments, options, lineno, content, state_machine=None, document=None, srcdir=None, builddir=None): """process :report: directive. *srdir* - top level directory of rst documents *builddir* - build directory """ tag = "%s:%i" % (str(document), lineno) logging.debug("report_directive.run: profile: started: rst: %s" % tag) # sort out the paths # reference is used for time-stamping tracker_name = directives.uri(arguments[0]) basedir, fname, basename, ext, outdir, codename, notebookname = Utils.buildPaths(tracker_name) # get the directory of the rst file rstdir, rstfile = os.path.split(document) # state_machine.document.attributes['source']) # root of document tree if srcdir is None: srcdir = setup.srcdir # build directory if builddir is None: builddir = setup.confdir # remove symbolic links srcdir, builddir, rstdir = [os.path.realpath(x) for x in (srcdir, builddir, rstdir)] # there are three directories: # builddir = directory where document is built in (usually _build/html or similar) # rstdir = directory where rst sources are located # srcdir = directory from which the build process is started # path to root relative to rst rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir) # path to root relative to rst rst2builddir = os.path.join(os.path.relpath(builddir, start=rstdir), outdir) # path relative to source (for images) root2builddir = os.path.join(os.path.relpath(builddir, start=srcdir), outdir) logging.debug( "report_directive.run: arguments=%s, options=%s, lineno=%s, content=%s, document=%s" % (str(arguments), str(options), str(lineno), str(content), str(document)) ) logging.debug( "report_directive.run: plotdir=%s, basename=%s, ext=%s, fname=%s, rstdir=%s, srcdir=%s, builddir=%s" % (tracker_name, basename, ext, fname, rstdir, srcdir, builddir) ) logging.debug( "report_directive.run: tracker_name=%s, basedir=%s, rst2src=%s, root2build=%s, outdir=%s, codename=%s" % (tracker_name, basedir, rst2srcdir, rst2builddir, outdir, codename) ) # try to create. If several processes try to create it, # testing with `if` will not work. try: os.makedirs(outdir) except OSError as msg: pass if not os.path.exists(outdir): raise OSError("could not create directory %s: %s" % (outdir, msg)) ######################################################## # collect options # replace placedholders try: options = Utils.updateOptions(options) except ValueError as msg: logging.warn("failure while updating options: %s" % msg) logging.debug("report_directive.run: options=%s" % (str(options),)) transformer_names = [] renderer_name = None # get layout option layout = options.get("layout", "column") option_map = getOptionMap() renderer_options = Utils.selectAndDeleteOptions(options, option_map["render"]) transformer_options = Utils.selectAndDeleteOptions(options, option_map["transform"]) dispatcher_options = Utils.selectAndDeleteOptions(options, option_map["dispatch"]) tracker_options = Utils.selectAndDeleteOptions(options, option_map["tracker"]) display_options = Utils.selectAndDeleteOptions(options, option_map["display"]) logging.debug("report_directive.run: renderer options: %s" % str(renderer_options)) logging.debug("report_directive.run: transformer options: %s" % str(transformer_options)) logging.debug("report_directive.run: dispatcher options: %s" % str(dispatcher_options)) logging.debug("report_directive.run: tracker options: %s" % str(tracker_options)) logging.debug("report_directive.run: display options: %s" % str(display_options)) if "transform" in display_options: transformer_names = display_options["transform"].split(",") del display_options["transform"] if "render" in display_options: renderer_name = display_options["render"] del display_options["render"] ######################################################## # check for missing files if renderer_name != None: options_key = ( str(renderer_options) + str(transformer_options) + str(dispatcher_options) + str(tracker_options) + str(transformer_names) ) options_hash = hashlib.md5(options_key.encode()).hexdigest() template_name = Utils.quote_filename(Config.SEPARATOR.join((tracker_name, renderer_name, options_hash))) filename_text = os.path.join(outdir, "%s.txt" % (template_name)) logging.debug("report_directive.run: options_hash=%s" % options_hash) ########################################################### # check for existing files # update strategy does not use file stamps, but checks # for presence/absence of text element and if all figures # mentioned in the text element are present ########################################################### queries = [re.compile("%s(%s\S+.%s)" % (root2builddir, outdir, suffix)) for suffix in ("png", "pdf")] logging.debug("report_directive.run: checking for changed files.") # check if text element exists if os.path.exists(filename_text): lines = [x[:-1] for x in open(filename_text, "r").readlines()] filenames = [] # check if all figures are present for line in lines: for query in queries: x = query.search(line) if x: filenames.extend(list(x.groups())) logging.debug("report_directive.run: %s: checking for %s" % (tag, str(filenames))) for filename in filenames: if not os.path.exists(filename): logging.info("report_directive.run: %s: redo: %s missing" % (tag, filename)) break else: logging.info("report_directive.run: %s: noredo: all files are present" % tag) ## all is present - save text and return if lines and state_machine: state_machine.insert_input(lines, state_machine.input_lines.source(0)) return [] else: logging.debug("report_directive.run: %s: no check performed: %s missing" % (tag, str(filename_text))) else: template_name = "" filename_text = None ########################################################## # Initialize collectors collectors = [] for collector in list(getPlugins("collect").values()): collectors.append(collector()) ########################################################## ## instantiate tracker, dispatcher, renderer and transformers ## and collect output ########################################################### try: ######################################################## # find the tracker logging.debug("report_directive.run: collecting tracker %s with options %s " % (tracker_name, tracker_options)) code, tracker = Utils.makeTracker(tracker_name, (), tracker_options) if not tracker: logging.error("report_directive.run: no tracker - no output from %s " % str(document)) raise ValueError("tracker `%s` not found" % tracker_name) logging.debug("report_directive.run: collected tracker %s" % tracker_name) tracker_id = Cache.tracker2key(tracker) ######################################################## # determine the transformer logging.debug("report_directive.run: creating transformers") transformers = Utils.getTransformers(transformer_names, transformer_options) ######################################################## # determine the renderer logging.debug("report_directive.run: creating renderer.") if renderer_name == None: logging.error("report_directive.run: no renderer - no output from %s" % str(document)) raise ValueError("the report directive requires a renderer") renderer = Utils.getRenderer(renderer_name, renderer_options) ######################################################## # create and call dispatcher logging.debug("report_directive.run: creating dispatcher") dispatcher = Dispatcher.Dispatcher(tracker, renderer, transformers) # add the tracker options dispatcher_options.update(tracker_options) blocks = dispatcher(**dispatcher_options) if blocks == None: blocks = ResultBlocks( ResultBlocks(Utils.buildWarning("NoData", "tracker %s returned no Data" % str(tracker))) ) code = None tracker_id = None except: logging.warn("report_directive.run: exception caught at %s:%i - see document" % (str(document), lineno)) blocks = ResultBlocks(ResultBlocks(Utils.buildException("invocation"))) code = None tracker_id = None logging.debug("report_directive.run: profile: started: collecting: %s" % tag) ######################################################## ## write code output linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename)) if code and basedir != outdir: with open(os.path.join(outdir, codename), "w") as outfile: for line in code: outfile.write(line) ######################################################## ## write notebook snippet linked_notebookname = re.sub("\\\\", "/", os.path.join(rst2srcdir, notebookname)) if basedir != outdir: with open(os.path.join(outdir, notebookname), "w") as outfile: Utils.writeNoteBookEntry( outfile, renderer=renderer_name, tracker=tracker_name, transformers=transformer_names, options=renderer_options.items() + tracker_options.items() + transformer_options.items(), ) ########################################################### # collect images ########################################################### map_figure2text = {} try: for collector in collectors: map_figure2text.update( collector.collect( blocks, template_name, outdir, rstdir, builddir, srcdir, content, display_options, tracker_id, links={"code_url": linked_codename, "notebook_url": linked_notebookname}, ) ) except: logging.warn( "report_directive.run: exception caught while collecting with %s at %s:%i - see document" % (collector, str(document), lineno) ) blocks = ResultBlocks(ResultBlocks(Utils.buildException("collection"))) code = None tracker_id = None ########################################################### # replace place holders or add text ########################################################### ## add default for text-only output map_figure2text["default-prefix"] = TEMPLATE_TEXT % locals() map_figure2text["default-suffix"] = "" blocks.updatePlaceholders(map_figure2text) ########################################################### ## render the output taking into account the layout lines = Utils.layoutBlocks(blocks, layout) ########################################################### # add caption lines.extend(["::", ""]) if content: lines.extend([" %s" % row.strip() for row in content]) lines.append("") lines.append("") # output rst text for this renderer if filename_text: outfile = open(filename_text, "w") outfile.write("\n".join(lines)) outfile.close() if SPHINXREPORT_DEBUG: for x, l in enumerate(lines): print("%5i %s" % (x, l)) if len(lines) and state_machine: state_machine.insert_input(lines, state_machine.input_lines.source(0)) logging.debug("report_directive.run: profile: finished: collecting: %s" % tag) logging.debug("report_directive.run: profile: finished: rst: %s:%i" % (str(document), lineno)) return []
def __call__(self, *args, **kwargs ): #self.debug( "%s: heap at start\n%s" % (self, str(HP.heap()) )) try: self.parseArguments( *args, **kwargs ) except: self.error( "%s: exception in parsing" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "parsing" ) )) # collect no data if tracker is the empty tracker # and go straight to rendering try: if self.tracker.getTracks() == ["empty"]: # is instance does not work because of module mapping # type(Tracker.Empty) == SphinxReport.Tracker.Empty # type(self.tracker) == Tracker.Empty # if isinstance( self.tracker, Tracker.Empty): result =self.renderer() return ResultBlocks( result ) except AttributeError: # for function trackers pass self.debug( "profile: started: tracker: %s" % (self.tracker)) # collecting data try: self.collect() except: self.error( "%s: exception in collection" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "collection" ) )) finally: self.debug( "profile: finished: tracker: %s" % (self.tracker)) if len(self.data) == 0: self.info( "%s: no data - processing complete" % self.tracker ) return None data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after collection: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # self.debug( "%s: heap after collection\n%s" % (self, str(HP.heap()) )) # transform data try: self.transform() except: self.error( "%s: exception in transformation" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "transformation" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after transformation: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # special Renderers - do not proceed # Special renderers if isinstance( self.renderer, Renderer.User): results = ResultBlocks( title="main" ) results.append( self.renderer( self.data, ('') ) ) return results elif isinstance( self.renderer, Renderer.Debug): results = ResultBlocks( title="main" ) results.append( self.renderer( self.data, ('') ) ) return results # self.debug( "%s: heap after transformation\n%s" % (self, str(HP.heap()) )) # restrict try: self.restrict() except: self.error( "%s: exception in restrict" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "restrict" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after restrict: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # exclude try: self.exclude() except: self.error( "%s: exception in exclude" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "exclude" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after exclude: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # remove superfluous levels try: self.prune() except: self.error( "%s: exception in pruning" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "pruning" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after pruning: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # remove group plots try: self.group() except: self.error( "%s: exception in grouping" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "grouping" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after grouping: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) self.debug( "profile: started: renderer: %s" % (self.renderer)) try: result = self.render() except: self.error( "%s: exception in rendering" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "rendering" ) )) finally: self.debug( "profile: finished: renderer: %s" % (self.renderer)) #self.debug( "%s: heap at end\n%s" % (self, str(HP.heap()) )) return result
def render( self ): '''supply the :class:`Renderer.Renderer` with the data to render. The data supplied will depend on the ``groupby`` option. returns a ResultBlocks data structure. ''' self.debug( "%s: rendering data started for %i items" % (self, len(self.data))) # get number of levels required by renderer try: renderer_nlevels = self.renderer.nlevels except AttributeError: # set to -1 to avoid any grouping # important for user renderers that are functions # and have no level attribute. renderer_nlevels = -1 # initiate output structure results = ResultBlocks( title = "") # convert to data series # The data is melted, i.e, # BMW price 10000 # BMW speed 100 # Golf price 5000 # Golf speed 50 dataframe = DataTree.asDataFrame( self.data ) # dataframe.write_csv( "test.csv" ) if dataframe is None: self.warn( "%s: no data after conversion" % self ) raise ValueError( "no data for renderer" ) # special patch: set column names to pruned levels # if there are no column names if len(dataframe.columns) == len(self.pruned): if list(dataframe.columns) == list(range( len(dataframe.columns))): dataframe.columns = [x[1] for x in self.pruned] index = dataframe.index def getIndexLevels( index ): try: # hierarchical index nlevels = len(index.levels) except AttributeError: nlevels = 1 index = [ (x,) for x in index] #raise ValueError('data frame without MultiIndex' ) return nlevels nlevels = getIndexLevels( index ) self.debug( "%s: rendering data started. levels=%i, required levels>=%i, group_level=%s" %\ (self, nlevels, renderer_nlevels, str(self.group_level) ) ) if renderer_nlevels < 0 and self.group_level <= 0: # no grouping for renderers that will accept # a dataframe with any level of indices and no explicit # grouping has been asked for. results.append( self.renderer( dataframe, path = () ) ) else: # user specified group level by default group_level = self.group_level # set group level to maximum allowed by renderer if renderer_nlevels >= 0: group_level = max(nlevels - renderer_nlevels, group_level) # add additional level if necessary if nlevels < group_level: prefix = tuple(["level%i" % x for x in range( group_level - nlevels)]) dataframe.index = pandas.MultiIndex.from_tuples( [ prefix + x for x in dataframe.index ] ) # used to be: group_level + 1 # hierarchical index # numpy.unique converts everything to a string # which is not consistent with selecting later paths = map( tuple, DataTree.unique( [ x[:group_level] for x in dataframe.index.unique() ] )) pathlength = len(paths[0]) - 1 is_hierarchical = isinstance( dataframe.index, pandas.core.index.MultiIndex ) if is_hierarchical: # Note: can only sort hierarchical indices dataframe = dataframe.sortlevel() if dataframe.index.lexsort_depth < pathlength: raise ValueError('could not sort data frame: sort depth=%i < pathlength=%i, dataframe=%s' \ % (dataframe.index.lexsort_depth, pathlength, dataframe)) for path in paths: if path: if len(path) == nlevels: # extract with loc in order to obtain dataframe work = dataframe.loc[[path]] else: # select data frame as cross-section work = dataframe.xs(path, axis=0 ) else: # empty tuple - use full data set work = dataframe # remove columns and rows in work that are all Na work = work.dropna( axis=1, how='all').dropna( axis=0, how='all') if is_hierarchical and renderer_nlevels >= 0: work_levels = getIndexLevels( work.index ) # reduce levels of indices required to that required # for Renderer. This occurs if groupby=none. if work_levels > renderer_nlevels: sep = work_levels - (renderer_nlevels - 1) tuples = [ ( DataTree.path2str( x[:sep] ), ) + x[sep:] \ for x in work.index ] work.index = pandas.MultiIndex.from_tuples( tuples ) try: results.append( self.renderer( work, path = path )) except: self.error( "%s: exception in rendering" % self ) results.append( ResultBlocks( Utils.buildException( "rendering" ) ) ) if len(results) == 0: self.warn("renderer returned no data.") raise ValueError( "renderer returned no data." ) self.debug( "%s: rendering data finished with %i blocks" % (self.tracker, len(results))) return results