def getTrackers(fullpath): """retrieve a tracker and its associated code. The tracker is not instantiated. returns a tuple (code, tracker, module, flag). The flag indicates whether that tracker is derived from the tracker base class. If False, that tracker will not be listed as an available tracker, though it might be specified at the command line. """ name, cls = os.path.splitext(fullpath) # remove leading '.' cls = cls[1:] module_name = os.path.basename(name) module, pathname = Utils.getModule(name) trackers = [] for name in dir(module): obj = getattr(module, name) try: if Utils.isClass(obj): trackers.append((name, obj, module_name, True)) else: trackers.append((name, obj, module_name, False)) except ValueError: pass return trackers
def __call__(self, data): if self.nlevels is None: # do not group return self.transform(data) dataframes, keys = [], [] group_levels = Utils.getGroupLevels( data, modify_levels=self.nlevels, ) for key, group in data.groupby(level=group_levels): self.debug('applying transformation on group %s' % str(key)) df = self.transform(group) if df is not None: dataframes.append(df) keys.append(key) df = pandas.concat(dataframes, keys=keys) if self.prune_dataframe: # reset dataframe index - keep the same levels Utils.pruneDataFrameIndex(df, original=data) self.debug("transform: finished") return df
def __call__(self, dataframe, path): # convert to dataframe # index has test names # columns are description, info, status columns = ('description', 'info', 'status', 'name') if set(dataframe.columns) != set(columns): raise ValueError("invalid columns: expected '%s', got '%s' " % (columns, dataframe.columns)) lines = [] dirname = os.path.join(os.path.dirname( sys.modules["CGATReport"].__file__), "images") descriptions = {} title = "status" # add header lines.append(".. csv-table:: %s" % "table") lines.append(' :header: "Track", "Test", "", "Status", "Info"') lines.append('') rows = [] for index, values in dataframe.iterrows(): testname = values['name'] status = values['status'] try: image = ".. image:: {}\n :width: 32".format( os.path.join(dirname, self.map_code2image[status.upper()])) except KeyError: image = "" rows.append({ "test": testname, "description": values["description"], "info": values['info'], "status": status, "track": path2str(index), "image": image, }) descriptions[testname] = values["description"] # filter and sort table table = [self.columns] table.extend([[row[x] for x in self.columns] for row in rows]) lines = Utils.table2rst(table).split("\n") if self.display_legend: lines.append(".. glossary::") lines.append("") for test, description in descriptions.items(): lines.append('%s\n%s\n' % (Utils.indent(test, 3), Utils.indent(description, 6))) return ResultBlocks(ResultBlock("\n".join(lines), title=""))
def __call__(self, dataframe, path): # convert to dataframe # index has test names # columns are description, info, status columns = ('description', 'info', 'status', 'name') if set(dataframe.columns) != set(columns): raise ValueError("invalid columns: expected '%s', got '%s' " % (columns, dataframe.columns)) lines = [] dirname = os.path.join(os.path.dirname( sys.modules["CGATReport"].__file__), "images") descriptions = {} title = "status" # add header lines.append(".. csv-table:: %s" % "table") lines.append(' :header: "Track", "Test", "", "Status", "Info"') lines.append('') for index, values in dataframe.iterrows(): testname = values['name'] description = values['description'] info = values['info'] status = values['status'] track = path2str(index) descriptions[testname] = description try: image = ".. image:: %s" %\ os.path.join(dirname, self.map_code2image[status.upper()]) except KeyError: image = "" lines.append( ' "%(track)s", ":term:`%(testname)s`", "%(image)s", "%(status)s", "%(info)s"' % locals()) lines.append("") lines.append(".. glossary::") lines.append("") for test, description in descriptions.items(): lines.append('%s\n%s\n' % (Utils.indent(test, 3), Utils.indent(description, 6))) return ResultBlocks(ResultBlock("\n".join(lines), title=""))
def collect(self, blocks, figure_key="", subfig=0): '''collect html output from result blocks. HTML output is written to a file and a link will be inserted at the place holder. ''' map_figure2text = {} extension = "html" for block in blocks: if not hasattr(block, "html"): continue # remove special characters from filename. I think the docutils # system removes these from links which later causes problems # as the link does not point to the correct location of the # file. outname = Utils.quote_filename( "%s_%s" % (self.template_name, block.title)) outputpath = os.path.join( self.outdir, '%s.%s' % (outname, extension)) # save to file outf = open(outputpath, "w") outf.write(block.html) outf.close() # use absolute path link = os.path.abspath(outputpath) rst_output = "%(link)s" % locals() map_figure2text["#$html %s$#" % block.title] = rst_output return map_figure2text
def collect(self, blocks, figure_key="", subfig=0): '''collect xls output from result blocks. xls output is written to a file and a link will be inserted at the place holder. ''' map_figure2text = {} extension = "xlsx" for block in blocks: if not hasattr(block, "xls"): continue outname = Utils.quote_filename( "%s_%s" % (self.template_name, block.title)) outputpath = os.path.join(self.outdir, '%s.%s' % (outname, extension)) # save to file block.xls.save(outputpath) # use absolute path link = os.path.abspath(outputpath) rst_output = ":download:`(link) </{}>`".format(link) map_figure2text["#$xls %s$#" % block.title] = rst_output return map_figure2text
def writeCode(class_name, code, inliner): '''write code of class to file. returns URI of written code. ''' document = inliner.document.current_source reference = class_name # root of document tree srcdir = setup.srcdir # get the directory of the rst file rstdir, rstfile = os.path.split(document) (basedir, fname, basename, ext, outdir, codename, notebookname) = Utils.build_paths(reference) # path to root relative to rst rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir) # output code linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename)) if code and basedir != outdir: outfile = open(os.path.join(outdir, codename), "w") for line in code: outfile.write(line) outfile.close() return linked_codename
def writeNotebook(outfile, options, kwargs, renderer_options, transformer_options, display_options, modulename, name): '''write a snippet to paste with the ipython notebook. ''' cmd_options = [ 'do_print = False', 'tracker="%s"' % options.tracker, 'renderer="%s"' % options.renderer, 'trackerdir="%s"' % options.trackerdir, 'workdir="%s"' % os.getcwd()] for key, val in list(kwargs.items()) +\ list(renderer_options.items()) +\ list(transformer_options.items()): if val is None: cmd_options.append("%s" % key) else: if Utils.isString(val): cmd_options.append('%s="%s"' % (key, val)) else: cmd_options.append('%s=%s' % (key, val)) if options.transformers: cmd_options.append( "transformer=['%s']" % "','".join(options.transformers)) # no module name in tracker params = {"tracker": "%s" % (name), "options": ",\n".join(cmd_options)} outfile.write(Utils.NOTEBOOK_TEXT_TEMPLATE % params)
def __call__(self, track, slice=None): # note there are spaces behind the %(image)s directive to accomodate # for path substitution block = ''' .. figure:: %(image)s :height: 300 ''' blocks = ResultBlocks() tracks = sorted([x.asFile() for x in TRACKS]) for track in tracks: files = glob.glob( os.path.join(EXPORTDIR, "fastqc", "%s*_fastqc" % track)) for x, fn in enumerate(sorted(files)): y = x + 1 image = os.path.abspath( os.path.join(fn, "Images", "%s.png" % slice)) if not os.path.exists(image): continue blocks.append(ResultBlock(text=block % locals(), title=os.path.basename(fn))) return odict((("rst", "\n".join(Utils.layoutBlocks( blocks, layout="columns-2"))),))
def __call__(self, track, slice=None): # note there are spaces behind the %(image)s directive to accomodate # for path substitution block = ''' .. figure:: %(image)s :height: 300 ''' blocks = ResultBlocks() tracks = sorted([x.asFile() for x in TRACKS]) for track in tracks: files = glob.glob( os.path.join(EXPORTDIR, "fastqc", "%s*_fastqc" % track)) for x, fn in enumerate(sorted(files)): y = x + 1 image = os.path.abspath( os.path.join(fn, "Images", "%s.png" % slice)) if not os.path.exists(image): continue blocks.append( ResultBlock(text=block % locals(), title=os.path.basename(fn))) return odict( (("rst", "\n".join(Utils.layoutBlocks(blocks, layout="columns-2"))), ))
def import_image(self, filename): """import image into report. The image is hard-linked. returns path to image for use in html. """ mangled_filename = re.sub("/", "_", filename) filename = os.path.abspath(filename) # directory in which to store images and thumbnails outdir = Utils.getOutputDirectory() image_filename = os.path.join(outdir, mangled_filename) # filenames to use in html - must take document hierarchy # into account rst2srcdir = os.path.join(os.path.relpath(self.src_dir, start=self.rst_dir), outdir) html_image_filename = os.path.join(rst2srcdir, mangled_filename) try: os.link(filename, image_filename) except OSError: # file exists pass return html_image_filename
def import_thumbnail(self, filename, thumbnail_size): '''import image in *filename* as a thumbnail. thumbnail_size is a tuple of (height, width) of the thumbnail in pixels. ''' mangled_filename = re.sub("/", "_", filename) # directory in which to store images and thumbnails outdir = Utils.getOutputDirectory() thumb_filename = os.path.join(outdir, "thumb-%s.png" % mangled_filename) image = PIL.Image.open(filename) image.thumbnail(thumbnail_size) image.save(thumb_filename) # filenames to use in html - must take document hierarchy # into account rst2srcdir = os.path.join( os.path.relpath(self.src_dir, start=self.rst_dir), outdir) html_thumb_filename = os.path.join(rst2srcdir, "thumb-%s.png" % mangled_filename) if self.build_environment: self.build_environment.dlfiles.add_file(self.rst_dir, html_thumb_filename) return html_thumb_filename
def collect(self, blocks, figure_key="", subfig=0): '''collect svg output from result blocks. SVG output is written to a file and an image will be inserted at the place holder. ''' map_figure2text = {} extension = "svg" rst_text = '''.. figure:: /{absfn} ''' for block in blocks: if not hasattr(block, "svg"): continue # remove special characters from filename. I think the docutils # system removes these from links which later causes problems # as the link does not point to the correct location of the # file. outname = Utils.quote_filename( "%s_%s" % (self.template_name, block.title)) outputpath = os.path.join( self.outdir, '%s.%s' % (outname, extension)) # save to file with open(outputpath, "w") as outf: outf.write(block.svg) # use absolute path absfn = os.path.abspath(outputpath) map_figure2text["#$svg %s$#" % block.title] = rst_text.format( absfn=absfn) return map_figure2text
def path2str(path): '''convert path to printable string.''' if path is None: return "" if Utils.isString(path): return path try: return "/".join(map(str, path)) except: return str(path)
def render(self, dataframe, path): self.startPlot() dataseries = Utils.toMultipleSeries(dataframe) names = [path2str(x[0]) for x in dataseries] data = [x[1] for x in dataseries] R.boxplot(data, names=names) return self.endPlot(dataframe, path)
def copy(src, dst): fn = os.path.join(dest, dst, os.path.basename(src)) if os.path.exists(fn): raise OSError("file %s already exists - not overwriting." % fn) outfile = open(fn, "wb") x = Utils.get_data("CGATReport", "templates/%s" % src) if len(x) == 0: raise ValueError('file %s is empty' % src) outfile.write(x) outfile.close()
def tracker2key(tracker): '''derive cache filename from a tracker.''' modulename = tracker.__module__ try: # works for functions (def) name = tracker.__name__ except AttributeError: # works for functors (class) name = tracker.__class__.__name__ return Utils.quote_filename(".".join((modulename, name)))
def __call__(self, track, slice=None): blocks = ResultBlocks() block = ''' .. figure:: %(image)s :height: 300 ''' for image in glob.glob(os.path.join(IMAGEDIR, "*.png")): blocks.append(ResultBlock(text=block % locals(), title="image")) return odict((("rst", "\n".join(Utils.layoutBlocks(blocks, layout="columns-2"))),))
def collect(self, blocks, figure_key="", subfig=0): '''collect holoview output. ''' map_figure2text = {} extension = "svg" rst_text = '''.. figure:: /{absfn} ''' for block in blocks: if not hasattr(block, "hv"): continue if self.engine == "mpl": renderer = hv.Store.renderers['matplotlib'].instance(fig='svg') plot = renderer.get_plot(block.hv) rendered = renderer(plot, fmt="auto") (data, info) = rendered # remove special characters from filename. I think the docutils # system removes these from links which later causes problems # as the link does not point to the correct location of the # file. outname = Utils.quote_filename( "%s_%s" % (self.template_name, block.title)) outputpath = os.path.join( self.outdir, '%s.%s' % (outname, extension)) # save to file with open(outputpath, "w") as outf: outf.write(data) # use absolute path absfn = os.path.abspath(outputpath) map_figure2text["#$hv %s$#" % block.title] = rst_text.format( absfn=absfn) elif self.engine == "bokeh": renderer = hv.renderer("bokeh") html = renderer.static_html(block.hv) lines = [".. only:: html\n"] +\ [" .. raw:: html\n"] +\ [" " + x for x in html.splitlines()] lines = "\n".join(lines) map_figure2text["#$hv %s$#" % block.title] = lines return map_figure2text
def setup(app): setup.app = app setup.config = app.config setup.confdir = app.confdir setup.srcdir = app.srcdir setup.builddir = os.getcwd() app.add_directive('report', report_directive) # update global parameters in Utils module. PARAMS = Utils.get_parameters() app.add_config_value('PARAMS', collections.defaultdict(), 'env') setup.logger = Component.get_logger() return {'parallel_read_safe': True}
def render(self, dataframe, path): self.startPlot() dataseries = Utils.toMultipleSeries(dataframe) # import pdb; pdb.set_trace() # rframe = pandas.rpy.common.convert_to_r_dataframe(dataframe) # R.boxplot(rframe) names = [path2str(x[0]) for x in dataseries] data = [x[1] for x in dataseries] R.boxplot(data, names=names) return self.endPlot(dataframe, path)
def getData(self, path): """get data for track and slice. Save data in persistent cache for further use. For functions, path should be an empty tuple. """ if path: key = DataTree.path2str(path) else: key = "all" result, fromcache = None, False # trackers with options are not cached if not self.nocache and not self.tracker_options: try: result = self.cache[key] fromcache = True except KeyError: pass except RuntimeError as msg: raise RuntimeError( "error when accessing key %s from cache: %s " "- potential problem with unpickable object?" % (key, msg)) kwargs = {} if self.tracker_options: kwargs = Utils.parse_tracker_options(self.tracker_options) if result is None: try: result = self.tracker(*path, **kwargs) except Exception as msg: self.warn("exception for tracker '%s', path '%s': msg=%s" % (str(self.tracker), DataTree.path2str( path), msg)) if VERBOSE: self.warn(traceback.format_exc()) raise # store in cache if not self.nocache and not fromcache: # exception - do not store data frames # test with None fails for some reason self.cache[key] = result return result
def transform(self, data): # check if data is melted: if len(data.columns) != 1: raise ValueError( 'transformer requires dataframe with ' 'a single column, got %s' % data.columns) column = data.columns[0] # iterate over lowest levels to build a dictionary of # sets genesets = {} nlevels = Utils.getDataFrameLevels(data) for key, group in data.groupby(level=list(range(nlevels))): if "background" in key and not self.background: continue genesets[key] = set(group[column]) values = [] if len(genesets) == 2: a = set(genesets[list(genesets.keys())[0]]) b = set(genesets[list(genesets.keys())[1]]) values.append(("10", len(a - b))) values.append(("01", len(b - a))) values.append(("11", len(a & b))) values.append( ("labels", list(map(path2str, list(genesets.keys()))))) elif len(genesets) == 3: a = set(genesets[list(genesets.keys())[0]]) b = set(genesets[list(genesets.keys())[1]]) c = set(genesets[list(genesets.keys())[2]]) values.append(("100", len(a - b - c))) values.append(("010", len(b - a - c))) values.append(("001", len(c - a - b))) values.append(("110", len((a & b) - c))) values.append(("101", len((a & c) - b))) values.append(("011", len((b & c) - a))) values.append(("111", len((a & b) & c))) values.append( ("labels", list(map(path2str, list(genesets.keys()))))) else: raise ValueError( "Can currently only cope with 2 or 3 way intersections") return DataTree.listAsDataFrame(values)
def param_role(role, rawtext, text, lineno, inliner, options={}, content=[]): '''inserts a member variable of a tracker class in the text.''' parts = text.split(".") if len(parts) < 2: msg = inliner.reporter.error( ':param: should be in class.value format ' ': "%s" is invalid.' % text, line=lineno) prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] class_name, parameter_name = ".".join(parts[:-1]), parts[-1] try: code, tracker, tracker_path = Utils.makeTracker(class_name) except AttributeError: tracker = None if not tracker: msg = inliner.reporter.error( ':param: can not find class ' '"%s".' % class_name, line=lineno) prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] try: value = getattr(tracker, parameter_name) except AttributeError: msg = inliner.reporter.error( ':param: can not find variable %s in ' ': "%s" is invalid -tracker=%s.' % (parameter_name, class_name, str(tracker)), line=lineno) prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] linked_codename = writeCode(class_name, code, inliner) node = nodes.reference(rawtext, utils.unescape(str(value)), refuri=linked_codename, **options) return [node], []
def _match(self, label, paths): '''return True if any of paths match to label.''' for s in paths: if label == s: return True elif s.startswith("r(") and s.endswith(")"): # collect pattern matches: # remove r() s = s[2:-1] # remove flanking quotation marks if s[0] in ('"', "'") and s[-1] in ('"', "'"): s = s[1:-1] rx = re.compile(s) if not Utils.isString(label): continue if rx.search(label): return True return False
def value_role(role, rawtext, text, lineno, inliner, options={}, content=[]): '''insert a single value from a tracker into text.''' class_name = text try: code, tracker, tracker_path = Utils.makeTracker(class_name) except (AttributeError, ImportError): tracker = None if not tracker: msg = inliner.reporter.error( ':value: can not find class ' '"%s".' % class_name, line=lineno) prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] # Python 2/3 try: value = str(tracker()) except TypeError as msg: print("python 3 problem: %s: tracker=%s" % (msg, str(tracker()))) linked_codename = writeCode(class_name, code, inliner) # Base URL mainly used by inliner.rfc_reference, so this is correct: # ref = inliner.document.settings.rfc_base_url + inliner.rfc_url % value # in example, but deprecated # set_classes(options) # node = nodes.literal(rawtext, # utils.unescape(str(value)), # **options) node = nodes.reference(rawtext, utils.unescape(str(value)), refuri=linked_codename, **options) return [node], []
def cleanTrackers(rst_files, options, args): '''instantiate trackers and get code.''' trackers = set() for f in rst_files: for lineno, b in getBlocksFromRstFile(f): trackers.add(b.mArguments[0]) ntested, ncleaned, nskipped = 0, 0, 0 for reference in trackers: try: code, tracker, tracker_path = report_directive.makeTracker( reference) except AttributeError: # ignore missing trackers nskipped += 1 continue new_codehash = hashlib.md5("".join(code)).hexdigest() (basedir, fname, basename, ext, outdir, codename, notebookname) = Utils.build_paths( reference) codefilename = os.path.join(outdir, codename) ntested += 1 if not os.path.exists(codefilename): nskipped += 1 continue old_codehash = hashlib.md5( "".join(open(codefilename, "r").readlines())).hexdigest() if new_codehash != old_codehash: removed = clean.removeTracker(reference) removed.extend(clean.removeText(reference)) print("code has changed for %s: %i files removed" % (reference, len(removed))) ncleaned += 1 print("CGATReport: %i Trackers changed (%i tested, %i skipped)" % (ncleaned, ntested, nskipped))
def writeCode(class_name, code, inliner): '''write code of class to file. returns URI of written code. ''' document = inliner.document.current_source reference = class_name # root of document tree srcdir = setup.srcdir # get the directory of the rst file rstdir, rstfile = os.path.split(document) (basedir, fname, basename, ext, outdir, codename, notebookname) = Utils.build_paths(reference) # path to root relative to rst rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir) # output code linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename)) if code and basedir != outdir: # patch, something is wrong with paths, files do not end up # in build directory, but in current directory because outdir # only has the last path components (_static/report_directive) if not os.path.exists(outdir): os.makedirs(outdir) outfile = open(os.path.join(outdir, codename), "w") for line in code: outfile.write(line) outfile.close() return linked_codename
def __call__(self, *args, **kwargs): try: self.parseArguments(*args, **kwargs) except: self.error("%s: exception in parsing" % self) return ResultBlocks(ResultBlocks(Utils.buildException("parsing"))) # collect no data if tracker is the empty tracker # and go straight to rendering try: if self.tracker.getTracks() == ["empty"]: # is instance does not work because of module mapping # type(Tracker.Empty) == CGATReport.Tracker.Empty # type(self.tracker) == Tracker.Empty # if isinstance(self.tracker, Tracker.Empty): result = self.renderer() return ResultBlocks(result) except AttributeError: # for function trackers pass self.debug("profile: started: tracker: %s" % (self.tracker)) # collecting data try: self.collect() except: self.error("%s: exception in collection" % self) return ResultBlocks(ResultBlocks( Utils.buildException("collection"))) finally: self.debug("profile: finished: tracker: %s" % (self.tracker)) if self.tree is None or len(self.tree) == 0: self.info("%s: no data - processing complete" % self.tracker) return None data_paths = DataTree.getPaths(self.tree) self.debug("%s: after collection: %i data_paths: %s" % (self, len(data_paths), str(data_paths))) # special Renderers - do not process data further but render # directly. Note that no transformations will be applied. if isinstance(self.renderer, Renderer.User): results = ResultBlocks(title="main") results.append(self.renderer(self.tree)) return results elif isinstance(self.renderer, Renderer.Debug): results = ResultBlocks(title="main") results.append(self.renderer(self.tree)) return results # merge all data to hierarchical indexed dataframe self.data = DataTree.asDataFrame(self.tree) self.debug("dataframe memory usage: total=%i,data=%i,index=%i,col=%i" % (self.data.values.nbytes + self.data.index.nbytes + self.data.columns.nbytes, self.data.values.nbytes, self.data.index.nbytes, self.data.columns.nbytes)) # if tracks are set by tracker, call tracker with dataframe if self.indexFromTracker: self.tracker.setIndex(self.data) # transform data try: self.transform() except: self.error("%s: exception in transformation" % self) return ResultBlocks(ResultBlocks( Utils.buildException("transformation"))) # data_paths = DataTree.getPaths(self.data) # self.debug("%s: after transformation: %i data_paths: %s" % # (self, len(data_paths), str(data_paths))) # restrict try: self.filterPaths(self.restrict_paths, mode="restrict") except: self.error("%s: exception in restrict" % self) return ResultBlocks(ResultBlocks( Utils.buildException("restrict"))) # data_paths = DataTree.getPaths(self.data) # self.debug("%s: after restrict: %i data_paths: %s" % # (self, len(data_paths), str(data_paths))) # exclude try: self.filterPaths(self.exclude_paths, mode="exclude") except: self.error("%s: exception in exclude" % self) return ResultBlocks(ResultBlocks(Utils.buildException("exclude"))) # data_paths = DataTree.getPaths(self.data) # self.debug("%s: after exclude: %i data_paths: %s" % # (self, len(data_paths), str(data_paths))) # No pruning - maybe enable later as a user option self.pruned = [] # try: # self.prune() # except: # self.error("%s: exception in pruning" % self) # return ResultBlocks(ResultBlocks(Utils.buildException("pruning"))) # data_paths = DataTree.getPaths(self.data) # self.debug("%s: after pruning: %i data_paths: %s" % # (self, len(data_paths), str(data_paths))) try: self.group() except: self.error("%s: exception in grouping" % self) return ResultBlocks(ResultBlocks(Utils.buildException("grouping"))) # data_paths = DataTree.getPaths(self.data) # self.debug("%s: after grouping: %i data_paths: %s" % # (self, len(data_paths), str(data_paths))) if self.renderer is not None: self.debug("profile: started: renderer: %s" % (self.renderer)) try: result = self.render() except: self.error("%s: exception in rendering" % self) return ResultBlocks(ResultBlocks( Utils.buildException("rendering"))) finally: self.debug("profile: finished: renderer: %s" % (self.renderer)) else: result = ResultBlocks(title="") return result
def render(self): '''supply the:class:`Renderer.Renderer` with the data to render. The data supplied will depend on the ``groupby`` option. returns a ResultBlocks data structure. ''' self.debug("%s: rendering data started for %i items" % (self, len(self.data))) # initiate output structure results = ResultBlocks(title="") dataframe = self.data # dataframe.write_csv("test.csv") if dataframe is None: self.warn("%s: no data after conversion" % self) raise ValueError("no data for renderer") # special patch: set column names to pruned levels # if there are no column names if len(dataframe.columns) == len(self.pruned): if list(dataframe.columns) == list(range(len(dataframe.columns))): dataframe.columns = [x[1] for x in self.pruned] nlevels = Utils.getDataFrameLevels(dataframe) self.debug("%s: rendering data started. " "levels=%i, group_level=%s" % (self, nlevels, str(self.group_level))) if self.group_level < 0: # no grouping for renderers that will accept # a dataframe with any level of indices and no explicit # grouping has been asked for. results.append(self.renderer(dataframe, path=())) else: level = Utils.getGroupLevels( dataframe, max_level=self.group_level+1) self.debug("%s: grouping by levels: %s" % (self, str(level))) for key, work in dataframe.groupby(level=level): try: results.append(self.renderer(work, path=key)) except: self.error("%s: exception in rendering" % self) results.append( ResultBlocks(Utils.buildException("rendering"))) if len(results) == 0: self.warn("renderer returned no data.") raise ValueError("renderer returned no data.") self.debug("%s: rendering data finished with %i blocks" % (self.tracker, len(results))) return results
def group(self): '''rearrange dataframe for desired grouping. Through grouping the dataframe is rearranged such that the level at which data will be grouped will be the first level in hierarchical index. If grouping by "track" is set, additional level will be added to ensure that grouping will happen. If grouping by "slice" is set, the first two levels will be swopped. The group level indicates at which level in the nested dictionary the data will be grouped with 0 indicating that everything will grouped together. ''' nlevels = Utils.getDataFrameLevels(self.data) try: default_level = self.renderer.group_level except AttributeError: # User rendere that is pure functions does not # have a group_level attribute default_level = 0 groupby = self.groupby if str(default_level).startswith("force"): groupby = default_level[len("force-"):] elif groupby == "default": groupby = default_level if groupby == "none": self.group_level = nlevels - 1 elif groupby == "track": self.group_level = 0 elif groupby == "slice": # rearrange first two levels in data tree if nlevels > 1: self.data = self.data.reorder_levels( [1, 0] + range(2, nlevels)) self.group_level = 0 elif groupby == "all": # group everything together self.group_level = -1 elif isinstance(groupby, int): # get group level from Renderer if groupby < 0: # negative levels - subtract from lowest # level g = nlevels + groupby - 1 else: g = groupby self.group_level = g else: self.group_level = 0 self.debug("grouping: nlevel=%i, groupby=%s, default=%s, group=%i" % (nlevels, groupby, str(default_level), self.group_level)) return self.data