def __call__(self, dataframe, path): result = ResultBlocks() texts = [] if self.head or self.tail: if self.head: texts.append(str(dataframe.head(self.head))) if self.tail: texts.append(str(dataframe.tail(self.tail))) elif self.summary: texts.append(str(dataframe.describe())) else: texts.append(str(dataframe)) # add indentation texts = ['\n'.join([' %s' % y for y in x.split('\n')]) for x in texts] formatted = ''' :: %s ''' % '\n ...\n'.join(texts) result.append(ResultBlock(formatted, title=path2str(path))) return result
def __call__(self, track, slice=None): edir = EXPORTDIR method = "utr_extension" blocks = ResultBlocks() filepath = "%(edir)s/%(method)s/%(track)s.readextension_%(region)s_%(direction)s.%(slice)s.png" block = \ ''' .. figure:: %(filename)s :height: 300 ''' # append spaces for file extension block = "\n".join([x + " " * 200 for x in block.split("\n")]) for region, direction in itertools.product(("downstream", "upstream"), ("sense", "antisense", "anysense")): filename = filepath % locals() if os.path.exists(filename): blocks.append(ResultBlock(text=block % locals(), title="%(track)s %(region)s %(direction)s" % locals())) # else: # blocks.append( ResultBlock( "", # title = "%(track)s %(region)s %(direction)s" % locals() ) ) return odict((("rst", "\n".join(Utils.layoutBlocks(blocks, layout="columns-3"))),))
def __call__(self, track, slice=None): edir = EXPORTDIR method = "utr_extension" blocks = ResultBlocks() filepath = "%(edir)s/%(method)s/%(track)s.readextension_%(region)s_%(direction)s.%(slice)s.png" block = \ ''' .. figure:: %(filename)s :height: 300 ''' # append spaces for file extension block = "\n".join([x + " " * 200 for x in block.split("\n")]) for region, direction in itertools.product( ("downstream", "upstream"), ("sense", "antisense", "anysense")): filename = filepath % locals() if os.path.exists(filename): blocks.append( ResultBlock(text=block % locals(), title="%(track)s %(region)s %(direction)s" % locals())) # else: # blocks.append( ResultBlock( "", # title = "%(track)s %(region)s %(direction)s" % locals() ) ) return odict( (("rst", "\n".join(Utils.layoutBlocks(blocks, layout="columns-3"))), ))
def __call__(self, track, slice=None): # note there are spaces behind the %(image)s directive to accomodate # for path substitution block = ''' .. figure:: %(image)s :height: 300 ''' blocks = ResultBlocks() tracks = sorted([x.asFile() for x in TRACKS]) for track in tracks: files = glob.glob( os.path.join(EXPORTDIR, "fastqc", "%s*_fastqc" % track)) for x, fn in enumerate(sorted(files)): y = x + 1 image = os.path.abspath( os.path.join(fn, "Images", "%s.png" % slice)) if not os.path.exists(image): continue blocks.append( ResultBlock(text=block % locals(), title=os.path.basename(fn))) return odict( (("rst", "\n".join(Utils.layoutBlocks(blocks, layout="columns-2"))), ))
def __call__(self, track, slice=None): # note there are spaces behind the %(image)s directive to accomodate # for path substitution block = ''' .. figure:: %(image)s :height: 300 ''' blocks = ResultBlocks() tracks = sorted([x.asFile() for x in TRACKS]) for track in tracks: files = glob.glob( os.path.join(EXPORTDIR, "fastqc", "%s*_fastqc" % track)) for x, fn in enumerate(sorted(files)): y = x + 1 image = os.path.abspath( os.path.join(fn, "Images", "%s.png" % slice)) if not os.path.exists(image): continue blocks.append(ResultBlock(text=block % locals(), title=os.path.basename(fn))) return odict((("rst", "\n".join(Utils.layoutBlocks( blocks, layout="columns-2"))),))
def render(self, data): # initiate output structure results = ResultBlocks(title='debug') try: results.append(ResultBlock(json.dumps( data, indent=4), title='')) except TypeError: results.append(ResultBlock(str(data), title='')) return results
def __call__(self, dataframe, path): results = ResultBlocks() if dataframe is None: return results title = path2str(path) row_headers = dataframe.index col_headers = dataframe.columns results.append(self.asSpreadSheet(dataframe, row_headers, col_headers, title)) return results
def __call__(self, track, slice=None): blocks = ResultBlocks() block = ''' .. figure:: %(image)s :height: 300 ''' for image in glob.glob(os.path.join(IMAGEDIR, "*.png")): blocks.append(ResultBlock(text=block % locals(), title="image")) return odict((("rst", "\n".join(Utils.layoutBlocks(blocks, layout="columns-2"))),))
def render(self, dataframe, path): blocks = ResultBlocks() options = self.get_slideshow_options() lines = [self.prefix % options] for title, row in dataframe.iterrows(): row = row[row.notnull()] values = row.tolist() headers = list(row.index) dataseries = dict(zip(headers, values)) try: # return value is a series filename = dataseries['filename'] except KeyError: self.warn( "no 'filename' key in path %s" % (path2str(path))) return blocks try: # return value is a series name = dataseries['name'] except KeyError: self.warn( "no 'name' key in path %s" % (path2str(path))) return blocks description, title = os.path.split(name) lines.extend(self.add_image(filename, title, description)) lines.append("""</div>""") lines.append(self.skin % options) lines.append("""</div>""") lines = "\n".join(lines).split("\n") lines = [".. only::html\n"] +\ [" .. raw:: html\n"] +\ [" " + x for x in lines] lines = "\n".join(lines) blocks.append(ResultBlock(text=lines, title=path2str(path))) return blocks
def render(self, data): # initiate output structure results = ResultBlocks(title='user') labels = DataTree.getPaths(data) # iterate over all items at leaf for path, branch in DataTree.getNodes(data, len(labels) - 2): for key in Utils.TrackerKeywords: if key in branch: # add a result block results.append(ResultBlock(branch[key], title=path2str(path))) return results
def render(self, work, path): """render the data. """ results = ResultBlocks(title=path) matrix, rows, columns = self.buildMatrix(work) title = path2str(path) if len(rows) == 0: return ResultBlocks(ResultBlock("", title=title)) # do not output large matrices as rst files # separate and force need to be mixed in. if self.separate or (not self.force and (len(rows) > self.max_rows or len(columns) > self.max_cols)): return ResultBlocks(self.asFile(pandas.DataFrame(matrix, index=rows, columns=columns), rows, columns, title), title=path) lines = [] lines.append(".. csv-table:: %s" % title) lines.append(' :header: "track","%s" ' % '","'.join(columns)) lines.append('') for row in range(len(rows)): lines.append( ' "%s","%s"' % (rows[row], '","'.join( [self.toString(x) for x in matrix[row]]))) lines.append("") if path is None: subtitle = "" else: subtitle = path2str(path) results.append(ResultBlock("\n".join(lines), title=subtitle)) return results
def endPlot(self, plots, legends, path): """close plots. """ result = ResultBlocks() title = path2str(path) for plot in plots: figid = plot._id lines = [] lines.append("") lines.append("#$bkh %s$#" % figid) lines.append("") r = ResultBlock("\n".join(lines), title=title) r.bokeh = plot result.append(r) return result
def __call__(self, dataframe, path): results = ResultBlocks() if dataframe is None: return results title = path2str(path) row_headers = dataframe.index col_headers = dataframe.columns # do not output large matrices as rst files if self.separate or (not self.force and (len(row_headers) > self.max_rows or len(col_headers) > self.max_cols)): if self.large == "xls": results.append(self.asSpreadSheet(dataframe, row_headers, col_headers, title)) else: results.append(self.asFile(dataframe, row_headers, col_headers, title)) results.append(self.asRST(dataframe, row_headers, col_headers, title)) return results
def __call__(self, dataframe, path): results = ResultBlocks() if dataframe is None: return results title = path2str(path) lines = [] lines.append(".. glossary::") lines.append("") for x, row in enumerate(dataframe.iterrows()): header, data = row txt = "\n ".join([x.strip() for x in str(data).split("\n")]) lines.append(' %s\n %s\n' % (path2str(header), txt)) lines.append("") results.append(ResultBlock("\n".join(lines), title=title)) return results
def __call__(self, dataframe, path): # modify table (adding/removing columns) according to user options # matrix, row_headers, col_headers = \ # self.modifyTable(matrix, row_headers, col_headers) dataframe = self.modifyTable(dataframe) title = path2str(path) results = ResultBlocks() row_headers = dataframe.index col_headers = dataframe.columns # as of sphinx 1.3.1, tables with more than 100 columns cause an # error: # Exception occurred: # File "/ifs/apps/apps/python-2.7.9/lib/python2.7/site-packages/docutils/writers/html4css1/__init__.py", line 642, in write_colspecs # colwidth = int(node['colwidth'] * 100.0 / width + 0.5) # ZeroDivisionError: float division by zero # # Thus, for table with more than 100 columns, force will be # disabled and max_cols set to a low value in order to make # sure the table is not displayed inline if len(col_headers) >= 90: self.force = False self.max_cols = 10 # do not output large matrices as rst files if self.separate or (not self.force and (len(row_headers) > self.max_rows or len(col_headers) > self.max_cols)): if self.large == "xls": results.append(self.asSpreadSheet(dataframe, row_headers, col_headers, title)) else: results.append(self.asFile(dataframe, row_headers, col_headers, title)) if self.preview: raise NotImplementedError('preview not implemented') row_headers = row_headers[:self.max_rows] col_headers = col_headers[:self.max_cols] # matrix = [x[:self.max_cols] for x in # matrix[:self.max_rows]] else: return results results.append(self.asCSV(dataframe, row_headers, col_headers, title)) return results
def __call__(self, dataframe, path): # modify table (adding/removing columns) according to user options # matrix, row_headers, col_headers = \ # self.modifyTable(matrix, row_headers, col_headers) dataframe = self.modifyTable(dataframe) title = path2str(path) results = ResultBlocks() row_headers = dataframe.index col_headers = dataframe.columns # do not output large matrices as rst files if self.separate or (not self.force and (len(row_headers) > self.max_rows or len(col_headers) > self.max_cols)): if self.large == "xls": results.append(self.asSpreadSheet(dataframe, row_headers, col_headers, title)) else: results.append(self.asFile(dataframe, row_headers, col_headers, title)) if self.preview: raise NotImplementedError('preview not implemented') row_headers = row_headers[:self.max_rows] col_headers = col_headers[:self.max_cols] # matrix = [x[:self.max_cols] for x in # matrix[:self.max_rows]] else: return results results.append(self.asCSV(dataframe, row_headers, col_headers, title)) return results
def __call__(self, *args, **kwargs): try: self.parseArguments(*args, **kwargs) except: self.error("%s: exception in parsing" % self) return ResultBlocks(ResultBlocks(Utils.buildException("parsing"))) # collect no data if tracker is the empty tracker # and go straight to rendering try: if self.tracker.getTracks() == ["empty"]: # is instance does not work because of module mapping # type(Tracker.Empty) == CGATReport.Tracker.Empty # type(self.tracker) == Tracker.Empty # if isinstance(self.tracker, Tracker.Empty): result = self.renderer() return ResultBlocks(result) except AttributeError: # for function trackers pass self.debug("profile: started: tracker: %s" % (self.tracker)) # collecting data try: self.collect() except: self.error("%s: exception in collection" % self) return ResultBlocks(ResultBlocks( Utils.buildException("collection"))) finally: self.debug("profile: finished: tracker: %s" % (self.tracker)) if self.tree is None or len(self.tree) == 0: self.info("%s: no data - processing complete" % self.tracker) return None data_paths = DataTree.getPaths(self.tree) self.debug("%s: after collection: %i data_paths: %s" % (self, len(data_paths), str(data_paths))) # special Renderers - do not process data further but render # directly. Note that no transformations will be applied. if isinstance(self.renderer, Renderer.User): results = ResultBlocks(title="main") results.append(self.renderer(self.tree)) return results elif isinstance(self.renderer, Renderer.Debug): results = ResultBlocks(title="main") results.append(self.renderer(self.tree)) return results # merge all data to hierarchical indexed dataframe self.data = DataTree.asDataFrame(self.tree) self.debug("dataframe memory usage: total=%i,data=%i,index=%i,col=%i" % (self.data.values.nbytes + self.data.index.nbytes + self.data.columns.nbytes, self.data.values.nbytes, self.data.index.nbytes, self.data.columns.nbytes)) # if tracks are set by tracker, call tracker with dataframe if self.indexFromTracker: self.tracker.setIndex(self.data) # transform data try: self.transform() except: self.error("%s: exception in transformation" % self) return ResultBlocks(ResultBlocks( Utils.buildException("transformation"))) # data_paths = DataTree.getPaths(self.data) # self.debug("%s: after transformation: %i data_paths: %s" % # (self, len(data_paths), str(data_paths))) # restrict try: self.filterPaths(self.restrict_paths, mode="restrict") except: self.error("%s: exception in restrict" % self) return ResultBlocks(ResultBlocks( Utils.buildException("restrict"))) # data_paths = DataTree.getPaths(self.data) # self.debug("%s: after restrict: %i data_paths: %s" % # (self, len(data_paths), str(data_paths))) # exclude try: self.filterPaths(self.exclude_paths, mode="exclude") except: self.error("%s: exception in exclude" % self) return ResultBlocks(ResultBlocks(Utils.buildException("exclude"))) # data_paths = DataTree.getPaths(self.data) # self.debug("%s: after exclude: %i data_paths: %s" % # (self, len(data_paths), str(data_paths))) # No pruning - maybe enable later as a user option self.pruned = [] # try: # self.prune() # except: # self.error("%s: exception in pruning" % self) # return ResultBlocks(ResultBlocks(Utils.buildException("pruning"))) # data_paths = DataTree.getPaths(self.data) # self.debug("%s: after pruning: %i data_paths: %s" % # (self, len(data_paths), str(data_paths))) try: self.group() except: self.error("%s: exception in grouping" % self) return ResultBlocks(ResultBlocks(Utils.buildException("grouping"))) # data_paths = DataTree.getPaths(self.data) # self.debug("%s: after grouping: %i data_paths: %s" % # (self, len(data_paths), str(data_paths))) if self.renderer is not None: self.debug("profile: started: renderer: %s" % (self.renderer)) try: result = self.render() except: self.error("%s: exception in rendering" % self) return ResultBlocks(ResultBlocks( Utils.buildException("rendering"))) finally: self.debug("profile: finished: renderer: %s" % (self.renderer)) else: result = ResultBlocks(title="") return result
def render(self): '''supply the:class:`Renderer.Renderer` with the data to render. The data supplied will depend on the ``groupby`` option. returns a ResultBlocks data structure. ''' self.debug("%s: rendering data started for %i items" % (self, len(self.data))) # initiate output structure results = ResultBlocks(title="") dataframe = self.data # dataframe.write_csv("test.csv") if dataframe is None: self.warn("%s: no data after conversion" % self) raise ValueError("no data for renderer") # special patch: set column names to pruned levels # if there are no column names if len(dataframe.columns) == len(self.pruned): if list(dataframe.columns) == list(range(len(dataframe.columns))): dataframe.columns = [x[1] for x in self.pruned] nlevels = Utils.getDataFrameLevels(dataframe) self.debug("%s: rendering data started. " "levels=%i, group_level=%s" % (self, nlevels, str(self.group_level))) if self.group_level < 0: # no grouping for renderers that will accept # a dataframe with any level of indices and no explicit # grouping has been asked for. results.append(self.renderer(dataframe, path=())) else: level = Utils.getGroupLevels( dataframe, max_level=self.group_level+1) self.debug("%s: grouping by levels: %s" % (self, str(level))) for key, work in dataframe.groupby(level=level): try: results.append(self.renderer(work, path=key)) except: self.error("%s: exception in rendering" % self) results.append( ResultBlocks(Utils.buildException("rendering"))) if len(results) == 0: self.warn("renderer returned no data.") raise ValueError("renderer returned no data.") self.debug("%s: rendering data finished with %i blocks" % (self.tracker, len(results))) return results
def layoutBlocks(blocks, layout="column"): """layout blocks of rst text. layout can be one of "column", "row", or "grid". The layout uses an rst table to arrange elements. """ lines = [] if len(blocks) == 0: return lines # flatten blocks bb = ResultBlocks() for b in blocks: if b.title: b.updateTitle(b.title, "prefix") try: bb.extend(b) except TypeError: bb.append(b) blocks = bb # check if postambles are identical across all blocks postambles = set([b.postamble for b in blocks]) if len(postambles) == 1: blocks.clearPostamble() postamble = postambles.pop() else: postamble = None if layout == "column": for block in blocks: if block.title: lines.extend(block.title.split("\n")) lines.append("") else: warn("report_directive.layoutBlocks: missing title") lines.extend(block.text.split("\n")) lines.extend(block.postamble.split("\n")) lines.append("") if postamble: lines.extend(postamble.split("\n")) lines.append("") return lines elif layout in ("row", "grid"): if layout == "row": ncols = len(blocks) elif layout == "grid": ncols = int(math.ceil(math.sqrt(len(blocks)))) elif layout.startswith("column"): ncols = min(len(blocks), int(layout.split("-")[1])) # TODO: think about appropriate fix for empty data if ncols == 0: ncols = 1 return lines else: raise ValueError("unknown layout %s " % layout) if ncols == 0: warn("no columns") return lines # compute column widths widths = [x.getWidth() for x in blocks] text_heights = [x.getTextHeight() for x in blocks] title_heights = [x.getTitleHeight() for x in blocks] columnwidths = [] for x in range(ncols): columnwidths.append(max([widths[y] for y in range(x, len(blocks), ncols)])) separator = "+%s+" % "+".join(["-" * x for x in columnwidths]) # add empty blocks if len(blocks) % ncols: blocks.extend([ResultBlock("", "")] * (ncols - len(blocks) % ncols)) for nblock in range(0, len(blocks), ncols): # add text lines.append(separator) max_height = max(text_heights[nblock:nblock + ncols]) new_blocks = ResultBlocks() for xx in range(nblock, min(nblock + ncols, len(blocks))): txt, col = blocks[xx].text.split("\n"), xx % ncols txt = blocks[xx].text.split("\n") + \ blocks[xx].postamble.split("\n") col = xx % ncols max_width = columnwidths[col] # add missig lines txt.extend([""] * (max_height - len(txt))) # extend lines txt = [x + " " * (max_width - len(x)) for x in txt] new_blocks.append(txt) for l in zip(*new_blocks): lines.append("|%s|" % "|".join(l)) # add subtitles max_height = max(title_heights[nblock:nblock + ncols]) if max_height > 0: new_blocks = ResultBlocks() lines.append(separator) for xx in range(nblock, min(nblock + ncols, len(blocks))): txt, col = blocks[xx].title.split("\n"), xx % ncols max_width = columnwidths[col] # add missig lines txt.extend([""] * (max_height - len(txt))) # extend lines txt = [x + " " * (max_width - len(x)) for x in txt] new_blocks.append(txt) for l in zip(*new_blocks): lines.append("|%s|" % "|".join(l)) lines.append(separator) if postamble: lines.append(postamble) lines.append("") return lines