def __call__(self, track, slice=None): # note there are spaces behind the %(image)s directive to accomodate # for path substitution block = ''' .. figure:: %(image)s :height: 300 ''' blocks = ResultBlocks() tracks = sorted([x.asFile() for x in TRACKS]) for track in tracks: files = glob.glob( os.path.join(EXPORTDIR, "fastqc", "%s*_fastqc" % track)) for x, fn in enumerate(sorted(files)): y = x + 1 image = os.path.abspath( os.path.join(fn, "Images", "%s.png" % slice)) if not os.path.exists(image): continue blocks.append( ResultBlock(text=block % locals(), title=os.path.basename(fn))) return odict( (("rst", "\n".join(Utils.layoutBlocks(blocks, layout="columns-2"))), ))
def __call__(self, track, slice = None ): edir = EXPORTDIR method = "utr_extension" blocks = ResultBlocks() filepath = "%(edir)s/%(method)s/%(track)s.readextension_%(region)s_%(direction)s.%(slice)s.png" block = \ ''' .. figure:: %(filename)s :height: 300 ''' # append spaces for file extension block = "\n".join( [ x + " " * 200 for x in block.split("\n") ] ) for region, direction in itertools.product( ("downstream", "upstream"), ("sense", "antisense", "anysense" )): filename = filepath % locals() if os.path.exists( filename ): blocks.append( ResultBlock( text = block % locals(), title = "%(track)s %(region)s %(direction)s" % locals() ) ) # else: # blocks.append( ResultBlock( "", # title = "%(track)s %(region)s %(direction)s" % locals() ) ) return odict( (("rst", "\n".join(Utils.layoutBlocks( blocks, layout="columns-3" ))),))
def __call__(self, track, slice=None): # note there are spaces behind the %(image)s directive to accomodate # for path substitution block = ''' .. figure:: %(image)s :height: 300 ''' blocks = ResultBlocks() tracks = sorted([x.asFile() for x in TRACKS]) for track in tracks: files = glob.glob( os.path.join(EXPORTDIR, "fastqc", "%s*_fastqc" % track)) for x, fn in enumerate(sorted(files)): y = x + 1 image = os.path.abspath( os.path.join(fn, "Images", "%s.png" % slice)) if not os.path.exists(image): continue blocks.append(ResultBlock(text=block % locals(), title=os.path.basename(fn))) return odict((("rst", "\n".join(Utils.layoutBlocks(blocks, layout="columns-2"))),))
def __call__(self, track, slice = None ): edir = EXPORTDIR method = "utr_extension" blocks = ResultBlocks() filepath = "%(edir)s/%(method)s/%(track)s.readextension_%(region)s_%(direction)s.%(slice)s.png" block = \ ''' .. figure:: %(filename)s :height: 300 ''' # append spaces for file extension block = "\n".join( [ x + " " * 200 for x in block.split("\n") ] ) for region, direction in itertools.product( ("downstream", "upstream"), ("sense", "antisense", "anysense" )): filename = filepath % locals() if os.path.exists( filename ): blocks.append( ResultBlock( text = block % locals(), title = "%(track)s %(region)s %(direction)s" % locals() ) ) # else: # blocks.append( ResultBlock( "", # title = "%(track)s %(region)s %(direction)s" % locals() ) ) return odict( (("rst", "\n".join(Utils.layoutBlocks( blocks, layout="columns-3" ))),))
def render( self, work, path ): # initiate output structure results = ResultBlocks( title = path ) # iterate over all items at leaf for key in work: t = type(work[key]) try: l = "%i" % len(work[key]) except AttributeError: l = "na" except TypeError: l = "na" # add a result block. data = str(work[key]) if len(data) > 30: data=data[:30] + "..." results.append( ResultBlock( "path= %s, type= %s, len= %s, data= %s" % \ ( path2str(path + (key,)), t, l, data), title = "") ) return results
def render( self, work, path ): """render the data. """ results = ResultBlocks( title = path ) matrix, rows, columns = self.buildMatrix( work ) title = path2str(path) if len(rows) == 0: return ResultBlocks( ResultBlock( "", title = title) ) # do not output large matrices as rst files if not self.force and (len(rows) > self.max_rows or len(columns) > self.max_cols): return self.asFile( [ [ self.toString(x) for x in r ] for r in matrix ], rows, columns, title ) lines = [] lines.append( ".. csv-table:: %s" % title ) lines.append( ' :header: "track","%s" ' % '","'.join( columns ) ) lines.append( '') for x in range(len(rows)): lines.append( ' "%s","%s"' % (rows[x], '","'.join( [ self.toString(x) for x in matrix[x,:] ] ) ) ) lines.append( "") if not path: subtitle = "" else: subtitle = path2str(path) results.append( ResultBlock( "\n".join(lines), title = subtitle ) ) return results
def __call__(self, data, path ): '''iterate over leaves/branches in data structure. This method will call the :meth:`render` method for each leaf/branch at level :attr:`nlevels`. ''' if self.nlevels == None: raise NotImplementedError("incomplete implementation of %s" % str(self)) result = ResultBlocks( title = path2str(path) ) labels = DataTree.getPaths( data ) if len(labels) < self.nlevels: self.warn( "at %s: expected at least %i levels - got %i: %s" %\ (str(path), self.nlevels, len(labels), str(labels)) ) result.append( EmptyResultBlock( title = path2str(path) ) ) return result paths = list(itertools.product( *labels[:-self.nlevels] )) for p in paths: work = DataTree.getLeaf( data, p ) if not work: continue try: result.extend( self.render( work, path + p ) ) except: self.warn("exeception raised in rendering for path: %s" % str(path+p)) raise return result
def render( self, data, path ): # initiate output structure results = ResultBlocks( title = path ) try: results.append( ResultBlock(json.dumps(data, indent=4), title='')) except TypeError: results.append( ResultBlock(str(data), title='')) return results
def __call__(self, track, slice = None ): blocks = ResultBlocks() block = ''' .. figure:: %(image)s :height: 300 ''' for image in glob.glob( os.path.join( IMAGEDIR, "*.png" )): blocks.append( ResultBlock( text = block % locals(), title = "image" ) ) return odict( (("rst", "\n".join( Utils.layoutBlocks( blocks, layout = "columns-2"))),))
def render( self, work, path ): # initiate output structure results = ResultBlocks( title = path2str(path) ) labels = DataTree.getPaths( work ) # iterate over all items at leaf for path, branch in DataTree.getNodes( work, len(labels) - 2 ): for key in Utils.TrackerKeywords: if key in branch: # add a result block results.append( ResultBlock( branch[key], title = path2str(path) ) ) return results
def render(self): """supply the :class:`Renderer.Renderer` with the data to render. The data supplied will depend on the ``groupby`` option. return resultblocks """ self.debug("%s: rendering data started for %i items" % (self, len(self.data))) results = ResultBlocks(title="main") # get number of levels required by renderer try: renderer_nlevels = self.renderer.nlevels except AttributeError: renderer_nlevels = 0 data_paths = DataTree.getPaths(self.data) nlevels = len(data_paths) group_level = self.group_level self.debug( "%s: rendering data started. levels=%i, required levels>=%i, group_level=%i, data_paths=%s" % (self, nlevels, renderer_nlevels, group_level, str(data_paths)[:100]) ) if nlevels < renderer_nlevels: # add some dummy levels if levels is not enough d = self.data for x in range(renderer_nlevels - nlevels): d = odict((("all", d),)) results.append(self.renderer(d, path=("all",))) elif group_level < 0 or renderer_nlevels < 0: # no grouping results.append(self.renderer(self.data, path=())) else: # group at level group_level paths = list(itertools.product(*data_paths[: group_level + 1])) for path in paths: work = DataTree.getLeaf(self.data, path) if not work: continue try: results.append(self.renderer(work, path=path)) except: results.append(ResultBlocks(Utils.buildException("rendering"))) if len(results) == 0: self.warn("tracker returned no data.") raise ValueError("tracker returned no data.") self.debug("%s: rendering data finished with %i blocks" % (self.tracker, len(results))) return results
def render( self, work, path ): # initiate output structure results = ResultBlocks( title = path ) # iterate over all items at leaf #for key in work: if "text" in work: # add a result block results.append( ResultBlock( work["text"], title = "" ) ) elif "rst" in work: # add a result block results.append( ResultBlock( work["rst"], title = "" ) ) return results
def render( self, work, path ): """render the data. """ results = ResultBlocks( title = path ) matrix, rows, columns = self.buildMatrix( work ) title = path2str(path) if len(rows) == 0: return ResultBlocks( ResultBlock( "", title = title) ) # do not output large matrices as rst files # separate and force need to be mixed in. if self.separate or (not self.force and (len(rows) > self.max_rows or len(columns) > self.max_cols)): return ResultBlocks( self.asFile( pandas.DataFrame( matrix, index = rows, columns = columns), rows, columns, title ), title = path ) lines = [] lines.append( ".. csv-table:: %s" % title ) lines.append( " :class: sortable" ) lines.append( ' :header: "track","%s" ' % '","'.join( columns ) ) lines.append( '') for x in range(len(rows)): lines.append( ' "%s","%s"' % (rows[x], '","'.join( [ self.toString(x) for x in matrix[x,:] ] ) ) ) lines.append( "") if path is None: subtitle = "" else: subtitle = path2str(path) results.append( ResultBlock( "\n".join(lines), title = subtitle ) ) return results
def __call__(self, dataframe, path): # modify table (adding/removing columns) according to user options # matrix, row_headers, col_headers = self.modifyTable( matrix, row_headers, col_headers ) dataframe = self.modifyTable( dataframe ) title = path2str(path) results = ResultBlocks() row_headers = dataframe.index col_headers = dataframe.columns # do not output large matrices as rst files if self.separate or (not self.force and (len(row_headers) > self.max_rows or len(col_headers) > self.max_cols)): if self.large == "xls": results.append( self.asSpreadSheet( dataframe, row_headers, col_headers, title ) ) else: results.append( self.asFile( dataframe, row_headers, col_headers, title ) ) if self.preview: row_headers = row_headers[:self.max_rows] col_headers = col_headers[:self.max_cols] matrix = [ x[:self.max_cols] for x in matrix[:self.max_rows] ] else: return results out = StringIO.StringIO() dataframe.to_csv( out ) lines = [] lines.append( ".. csv-table:: %s" % title ) lines.append( " :class: sortable" ) if self.add_rowindex: lines.append( ' :header: "row", "", "%s" ' % '","'.join( map(str, col_headers) ) ) lines.append( '' ) x = 0 for header, line in zip( row_headers, matrix ): x += 1 lines.append( ' %i,"%s","%s"' % (x, str(header), '","'.join( map(str, line) ) ) ) else: l = out.getvalue().split("\n") lines.append( ' :header: %s' % l[0] ) lines.append( '' ) lines.extend( [' %s' % x for x in l[1:] ] ) lines.append( "") results.append( ResultBlock( "\n".join(lines), title = title) ) return results
def layoutBlocks( blocks, layout = "column"): """layout blocks of rst text. layout can be one of "column", "row", or "grid". The layout uses an rst table to arrange elements. """ lines = [] if len(blocks) == 0: return lines # flatten blocks x = ResultBlocks() for b in blocks: if b.title: b.updateTitle( b.title, "prefix" ) try: x.extend( b ) except TypeError: x.append( b ) blocks = x if layout == "column": for block in blocks: if block.title: lines.extend( block.title.split("\n") ) lines.append( "" ) else: logging.warn( "report_directive.layoutBlocks: missing title" ) lines.extend(block.text.split("\n")) lines.extend( [ "", ] ) return lines elif layout in ("row", "grid"): if layout == "row": ncols = len(blocks) elif layout == "grid": ncols = int(math.ceil(math.sqrt( len(blocks) ))) elif layout.startswith("column"): ncols = min( len(blocks), int(layout.split("-")[1])) # TODO: think about appropriate fix for empty data if ncols == 0: ncols = 1 return lines else: raise ValueError( "unknown layout %s " % layout ) # compute column widths widths = [ x.getWidth() for x in blocks ] text_heights = [ x.getTextHeight() for x in blocks ] title_heights = [ x.getTitleHeight() for x in blocks ] columnwidths = [] for x in range(ncols): columnwidths.append( max( [widths[y] for y in range( x, len(blocks), ncols ) ] ) ) separator = "+%s+" % "+".join( ["-" * x for x in columnwidths ] ) ## add empty blocks if len(blocks) % ncols: blocks.extend( [ ResultBlock( "", "" )] * (ncols - len(blocks) % ncols) ) for nblock in range(0, len(blocks), ncols ): ## add text lines.append( separator ) max_height = max( text_heights[nblock:nblock+ncols] ) new_blocks = ResultBlocks() for xx in range(nblock, min(nblock+ncols,len(blocks))): txt, col = blocks[xx].text.split("\n"), xx % ncols max_width = columnwidths[col] # add missig lines txt.extend( [""] * (max_height - len(txt)) ) # extend lines txt = [ x + " " * (max_width - len(x)) for x in txt ] new_blocks.append( txt ) for l in zip( *new_blocks ): lines.append( "|%s|" % "|".join( l ) ) ## add subtitles max_height = max( title_heights[nblock:nblock+ncols] ) if max_height > 0: new_blocks = ResultBlocks() lines.append( separator ) for xx in range(nblock, min(nblock+ncols,len(blocks))): txt, col = blocks[xx].title.split("\n"), xx % ncols max_width = columnwidths[col] # add missig lines txt.extend( [""] * (max_height - len(txt) ) ) # extend lines txt = [ x + " " * (max_width - len(x)) for x in txt ] new_blocks.append( txt ) for l in zip( *new_blocks ): lines.append( "|%s|" % "|".join( l ) ) lines.append( separator ) lines.append( "" ) return lines
def __call__(self, *args, **kwargs ): #self.debug( "%s: heap at start\n%s" % (self, str(HP.heap()) )) try: self.parseArguments( *args, **kwargs ) except: self.error( "%s: exception in parsing" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "parsing" ) )) # collect no data if tracker is the empty tracker # and go straight to rendering try: if self.tracker.getTracks() == ["empty"]: # is instance does not work because of module mapping # type(Tracker.Empty) == SphinxReport.Tracker.Empty # type(self.tracker) == Tracker.Empty # if isinstance( self.tracker, Tracker.Empty): result =self.renderer() return ResultBlocks( result ) except AttributeError: # for function trackers pass self.debug( "profile: started: tracker: %s" % (self.tracker)) # collecting data try: self.collect() except: self.error( "%s: exception in collection" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "collection" ) )) finally: self.debug( "profile: finished: tracker: %s" % (self.tracker)) if len(self.data) == 0: self.info( "%s: no data - processing complete" % self.tracker ) return None data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after collection: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # self.debug( "%s: heap after collection\n%s" % (self, str(HP.heap()) )) # transform data try: self.transform() except: self.error( "%s: exception in transformation" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "transformation" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after transformation: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # special Renderers - do not proceed # Special renderers if isinstance( self.renderer, Renderer.User): results = ResultBlocks( title="main" ) results.append( self.renderer( self.data, ('') ) ) return results elif isinstance( self.renderer, Renderer.Debug): results = ResultBlocks( title="main" ) results.append( self.renderer( self.data, ('') ) ) return results # self.debug( "%s: heap after transformation\n%s" % (self, str(HP.heap()) )) # restrict try: self.restrict() except: self.error( "%s: exception in restrict" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "restrict" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after restrict: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # exclude try: self.exclude() except: self.error( "%s: exception in exclude" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "exclude" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after exclude: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # remove superfluous levels try: self.prune() except: self.error( "%s: exception in pruning" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "pruning" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after pruning: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # remove group plots try: self.group() except: self.error( "%s: exception in grouping" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "grouping" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after grouping: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) self.debug( "profile: started: renderer: %s" % (self.renderer)) try: result = self.render() except: self.error( "%s: exception in rendering" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "rendering" ) )) finally: self.debug( "profile: finished: renderer: %s" % (self.renderer)) #self.debug( "%s: heap at end\n%s" % (self, str(HP.heap()) )) return result
def render( self ): '''supply the :class:`Renderer.Renderer` with the data to render. The data supplied will depend on the ``groupby`` option. returns a ResultBlocks data structure. ''' self.debug( "%s: rendering data started for %i items" % (self, len(self.data))) # get number of levels required by renderer try: renderer_nlevels = self.renderer.nlevels except AttributeError: # set to -1 to avoid any grouping # important for user renderers that are functions # and have no level attribute. renderer_nlevels = -1 # initiate output structure results = ResultBlocks( title = "") # convert to data series # The data is melted, i.e, # BMW price 10000 # BMW speed 100 # Golf price 5000 # Golf speed 50 dataframe = DataTree.asDataFrame( self.data ) # dataframe.write_csv( "test.csv" ) if dataframe is None: self.warn( "%s: no data after conversion" % self ) raise ValueError( "no data for renderer" ) # special patch: set column names to pruned levels # if there are no column names if len(dataframe.columns) == len(self.pruned): if list(dataframe.columns) == list(range( len(dataframe.columns))): dataframe.columns = [x[1] for x in self.pruned] index = dataframe.index def getIndexLevels( index ): try: # hierarchical index nlevels = len(index.levels) except AttributeError: nlevels = 1 index = [ (x,) for x in index] #raise ValueError('data frame without MultiIndex' ) return nlevels nlevels = getIndexLevels( index ) self.debug( "%s: rendering data started. levels=%i, required levels>=%i, group_level=%s" %\ (self, nlevels, renderer_nlevels, str(self.group_level) ) ) if renderer_nlevels < 0 and self.group_level <= 0: # no grouping for renderers that will accept # a dataframe with any level of indices and no explicit # grouping has been asked for. results.append( self.renderer( dataframe, path = () ) ) else: # user specified group level by default group_level = self.group_level # set group level to maximum allowed by renderer if renderer_nlevels >= 0: group_level = max(nlevels - renderer_nlevels, group_level) # add additional level if necessary if nlevels < group_level: prefix = tuple(["level%i" % x for x in range( group_level - nlevels)]) dataframe.index = pandas.MultiIndex.from_tuples( [ prefix + x for x in dataframe.index ] ) # used to be: group_level + 1 # hierarchical index # numpy.unique converts everything to a string # which is not consistent with selecting later paths = map( tuple, DataTree.unique( [ x[:group_level] for x in dataframe.index.unique() ] )) pathlength = len(paths[0]) - 1 is_hierarchical = isinstance( dataframe.index, pandas.core.index.MultiIndex ) if is_hierarchical: # Note: can only sort hierarchical indices dataframe = dataframe.sortlevel() if dataframe.index.lexsort_depth < pathlength: raise ValueError('could not sort data frame: sort depth=%i < pathlength=%i, dataframe=%s' \ % (dataframe.index.lexsort_depth, pathlength, dataframe)) for path in paths: if path: if len(path) == nlevels: # extract with loc in order to obtain dataframe work = dataframe.loc[[path]] else: # select data frame as cross-section work = dataframe.xs(path, axis=0 ) else: # empty tuple - use full data set work = dataframe # remove columns and rows in work that are all Na work = work.dropna( axis=1, how='all').dropna( axis=0, how='all') if is_hierarchical and renderer_nlevels >= 0: work_levels = getIndexLevels( work.index ) # reduce levels of indices required to that required # for Renderer. This occurs if groupby=none. if work_levels > renderer_nlevels: sep = work_levels - (renderer_nlevels - 1) tuples = [ ( DataTree.path2str( x[:sep] ), ) + x[sep:] \ for x in work.index ] work.index = pandas.MultiIndex.from_tuples( tuples ) try: results.append( self.renderer( work, path = path )) except: self.error( "%s: exception in rendering" % self ) results.append( ResultBlocks( Utils.buildException( "rendering" ) ) ) if len(results) == 0: self.warn("renderer returned no data.") raise ValueError( "renderer returned no data." ) self.debug( "%s: rendering data finished with %i blocks" % (self.tracker, len(results))) return results