def __call__(self, data ): if self.nlevels == None: raise NotImplementedError("incomplete implementation of %s" % str(self)) labels = DataTree.getPaths( data ) debug( "transform: started with paths: %s" % labels) assert len(labels) >= self.nlevels, "expected at least %i levels - got %i" % (self.nlevels, len(labels)) paths = list(itertools.product( *labels[:-self.nlevels] )) for path in paths: work = DataTree.getLeaf( data, path ) if not work: continue new_data = self.transform( work, path ) if new_data: if path: DataTree.setLeaf( data, path, new_data ) else: # set new root data = new_data else: warn( "no data at %s - removing branch" % str(path)) DataTree.removeLeaf( data, path ) debug( "transform: finished with paths: %s" % DataTree.getPaths( data )) return data
def __call__(self, *args, **kwargs): try: self.parseArguments(*args, **kwargs) except: return ResultBlocks(ResultBlocks(Utils.buildException("parsing"))) self.debug("profile: started: tracker: %s" % (self.tracker)) try: self.collect() except: return ResultBlocks(ResultBlocks(Utils.buildException("collection"))) self.debug("profile: finished: tracker: %s" % (self.tracker)) data_paths = DataTree.getPaths(self.data) self.debug("%s: after collection: %i data_paths: %s" % (self, len(data_paths), str(data_paths))) # transform data try: self.transform() except: return ResultBlocks(ResultBlocks(Utils.buildException("transformation"))) data_paths = DataTree.getPaths(self.data) self.debug("%s: after transformation: %i data_paths: %s" % (self, len(data_paths), str(data_paths))) # remove superfluous levels try: self.prune() except: return ResultBlocks(ResultBlocks(Utils.buildException("pruning"))) data_paths = DataTree.getPaths(self.data) self.debug("%s: after pruning: %i data_paths: %s" % (self, len(data_paths), str(data_paths))) # remove group plots try: self.group() except: return ResultBlocks(ResultBlocks(Utils.buildException("grouping"))) data_paths = DataTree.getPaths(self.data) self.debug("%s: after grouping: %i data_paths: %s" % (self, len(data_paths), str(data_paths))) self.debug("profile: started: renderer: %s" % (self.renderer)) try: result = self.render() except: return ResultBlocks(ResultBlocks(Utils.buildException("rendering"))) self.debug("profile: finished: renderer: %s" % (self.renderer)) return result
def exclude( self ): '''exclude data paths. Only those data paths not matching the exclude term are accepted. ''' if not self.exclude_paths: return data_paths = DataTree.getPaths( self.data ) # currently enumerates - bfs more efficient all_paths = list(itertools.product( *data_paths )) for path in all_paths: for s in self.exclude_paths: if s in path: self.debug( "%s: ignoring path %s because of :exclude:=%s" % (self.tracker, path, s)) try: DataTree.removeLeaf( self.data, path ) except KeyError: pass elif s.startswith("r(") and s.endswith(")"): # collect pattern matches: # remove r() s = s[2:-1] # remove flanking quotation marks if s[0] in ('"', "'") and s[-1] in ('"', "'"): s = s[1:-1] rx = re.compile( s ) if any( ( rx.search( p ) for p in path ) ): self.debug( "%s: ignoring path %s because of :exclude:=%s" % (self.tracker, path, s)) try: DataTree.removeLeaf( self.data, path ) except KeyError: pass
def __call__(self, data, path ): '''iterate over leaves/branches in data structure. This method will call the :meth:`render` method for each leaf/branch at level :attr:`nlevels`. ''' if self.nlevels == None: raise NotImplementedError("incomplete implementation of %s" % str(self)) result = ResultBlocks( title = path2str(path) ) labels = DataTree.getPaths( data ) if len(labels) < self.nlevels: self.warn( "at %s: expected at least %i levels - got %i: %s" %\ (str(path), self.nlevels, len(labels), str(labels)) ) result.append( EmptyResultBlock( title = path2str(path) ) ) return result paths = list(itertools.product( *labels[:-self.nlevels] )) for p in paths: work = DataTree.getLeaf( data, p ) if not work: continue try: result.extend( self.render( work, path + p ) ) except: self.warn("exeception raised in rendering for path: %s" % str(path+p)) raise return result
def render(self): """supply the :class:`Renderer.Renderer` with the data to render. The data supplied will depend on the ``groupby`` option. return resultblocks """ self.debug("%s: rendering data started for %i items" % (self, len(self.data))) results = ResultBlocks(title="main") # get number of levels required by renderer try: renderer_nlevels = self.renderer.nlevels except AttributeError: renderer_nlevels = 0 data_paths = DataTree.getPaths(self.data) nlevels = len(data_paths) group_level = self.group_level self.debug( "%s: rendering data started. levels=%i, required levels>=%i, group_level=%i, data_paths=%s" % (self, nlevels, renderer_nlevels, group_level, str(data_paths)[:100]) ) if nlevels < renderer_nlevels: # add some dummy levels if levels is not enough d = self.data for x in range(renderer_nlevels - nlevels): d = odict((("all", d),)) results.append(self.renderer(d, path=("all",))) elif group_level < 0 or renderer_nlevels < 0: # no grouping results.append(self.renderer(self.data, path=())) else: # group at level group_level paths = list(itertools.product(*data_paths[: group_level + 1])) for path in paths: work = DataTree.getLeaf(self.data, path) if not work: continue try: results.append(self.renderer(work, path=path)) except: results.append(ResultBlocks(Utils.buildException("rendering"))) if len(results) == 0: self.warn("tracker returned no data.") raise ValueError("tracker returned no data.") self.debug("%s: rendering data finished with %i blocks" % (self.tracker, len(results))) return results
def prune(self): """prune data tree. Remove all empty leaves. Remove all levels from the data tree that are superfluous, i.e. levels that contain only a single label all labels in the hierarchy below are the same. Ignore both the first and last level for this analyis. """ # remove all empty leaves DataTree.removeEmptyLeaves(self.data) # prune superfluous levels data_paths = DataTree.getPaths(self.data) nlevels = len(data_paths) # get number of levels required by renderer try: renderer_nlevels = self.renderer.nlevels except AttributeError: renderer_nlevels = 0 # do not prune for renderers that want all data if renderer_nlevels < 0: return levels_to_prune = [] for level in range(1, nlevels - 1): # check for single label in level if len(data_paths[level]) == 1: label = data_paths[level][0] prefixes = DataTree.getPrefixes(self.data, level) keep = False for prefix in prefixes: leaves = DataTree.getLeaf(self.data, prefix) if len(leaves) > 1 or label not in leaves: keep = True break if not keep: levels_to_prune.append((level, label)) levels_to_prune.reverse() # only prune to the minimum of levels required by renderer at most # levels_to_prune = levels_to_prune[:nlevels - renderer_nlevels] for level, label in levels_to_prune: self.debug("pruning level %i from data tree: label='%s'" % (level, label)) DataTree.removeLevel(self.data, level)
def group( self ): '''rearrange data tree for grouping and set group level. Through grouping the data tree is rearranged such that the level at which data will be grouped will be the top (0th) level in the nested dictionary. ''' data_paths = DataTree.getPaths( self.data ) nlevels = len(data_paths) # get number of levels required by renderer try: renderer_nlevels = self.renderer.nlevels except AttributeError: renderer_nlevels = 0 if self.groupby == "none": self.group_level = nlevels - 1 elif self.groupby == "track": # track is first level self.group_level = 1 # add pseudo levels, if there are not enough levels # to group by track if nlevels == renderer_nlevels: d = odict() for x in data_paths[0]: d[x] = odict( ((x, self.data[x]),)) self.data = d elif self.groupby == "slice": # rearrange tracks and slices in data tree if nlevels <= 2 : self.warn( "grouping by slice, but only %i levels in data tree - all are grouped" % nlevels) self.group_level = 0 else: self.data = DataTree.swop( self.data, 0, 1) self.group_level = 1 elif self.groupby == "all": # group everything together self.group_level = 0 else: # neither group by slice or track ("ungrouped") self.group_level = 0 return self.data
def render( self, work, path ): # initiate output structure results = ResultBlocks( title = path2str(path) ) labels = DataTree.getPaths( work ) # iterate over all items at leaf for path, branch in DataTree.getNodes( work, len(labels) - 2 ): for key in Utils.TrackerKeywords: if key in branch: # add a result block results.append( ResultBlock( branch[key], title = path2str(path) ) ) return results
def transform(self,data,path): from rpy2.robjects import r as R paths, lengths, values = [],[],[] labels = DataTree.getPaths(data) paths = list(itertools.product( *labels[:-1])) for path in paths: work = DataTree.getLeaf(data, path) try: lengths.append(len(work[self.pval])) values.extend(work[self.pval]) except TypeError: lengths.append(0) values.append(work[self.pval]) padj = R["p.adjust"](values, method = self.method) padj = [x for x in padj] for path in paths: num = lengths.pop(0) if num > 0: new_values = padj[0:num] padj = padj[num:] else: new_values = padj[0] padj = padj[1:] if path: work = odict(DataTree.getLeaf(data,path)) work["P-adjust"] = new_values DataTree.setLeaf(data,path,work) else: data["P-adjust"] = new_values return data
def group(self): """rearrange data tree for grouping. and set group level. """ data_paths = DataTree.getPaths(self.data) nlevels = len(data_paths) # get number of levels required by renderer try: renderer_nlevels = self.renderer.nlevels except AttributeError: renderer_nlevels = 0 if self.groupby == "none": self.group_level = renderer_nlevels elif self.groupby == "track": # track is first level self.group_level = 0 # add pseudo levels, if there are not enough levels # to group by track if nlevels == renderer_nlevels: d = odict() for x in data_paths[0]: d[x] = odict(((x, self.data[x]),)) self.data = d elif self.groupby == "slice": # rearrange tracks and slices in data tree if nlevels <= 2: warn("grouping by slice, but only %i levels in data tree - all are grouped" % nlevels) self.group_level = -1 else: self.data = DataTree.swop(self.data, 0, 1) self.group_level = 0 elif self.groupby == "all": # group everthing together self.group_level = -1 else: # neither group by slice or track ("ungrouped") self.group_level = -1
def __call__ (self, data): if self.nlevels == 0: self.nlevels = len(DataTree.getPaths(data)) return Transformer.__call__(self,data)
def __call__(self, *args, **kwargs ): #self.debug( "%s: heap at start\n%s" % (self, str(HP.heap()) )) try: self.parseArguments( *args, **kwargs ) except: self.error( "%s: exception in parsing" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "parsing" ) )) # collect no data if tracker is the empty tracker # and go straight to rendering try: if self.tracker.getTracks() == ["empty"]: # is instance does not work because of module mapping # type(Tracker.Empty) == SphinxReport.Tracker.Empty # type(self.tracker) == Tracker.Empty # if isinstance( self.tracker, Tracker.Empty): result =self.renderer() return ResultBlocks( result ) except AttributeError: # for function trackers pass self.debug( "profile: started: tracker: %s" % (self.tracker)) # collecting data try: self.collect() except: self.error( "%s: exception in collection" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "collection" ) )) finally: self.debug( "profile: finished: tracker: %s" % (self.tracker)) if len(self.data) == 0: self.info( "%s: no data - processing complete" % self.tracker ) return None data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after collection: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # self.debug( "%s: heap after collection\n%s" % (self, str(HP.heap()) )) # transform data try: self.transform() except: self.error( "%s: exception in transformation" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "transformation" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after transformation: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # special Renderers - do not proceed # Special renderers if isinstance( self.renderer, Renderer.User): results = ResultBlocks( title="main" ) results.append( self.renderer( self.data, ('') ) ) return results elif isinstance( self.renderer, Renderer.Debug): results = ResultBlocks( title="main" ) results.append( self.renderer( self.data, ('') ) ) return results # self.debug( "%s: heap after transformation\n%s" % (self, str(HP.heap()) )) # restrict try: self.restrict() except: self.error( "%s: exception in restrict" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "restrict" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after restrict: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # exclude try: self.exclude() except: self.error( "%s: exception in exclude" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "exclude" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after exclude: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # remove superfluous levels try: self.prune() except: self.error( "%s: exception in pruning" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "pruning" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after pruning: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) # remove group plots try: self.group() except: self.error( "%s: exception in grouping" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "grouping" ) )) data_paths = DataTree.getPaths( self.data ) self.debug( "%s: after grouping: %i data_paths: %s" % (self,len(data_paths), str(data_paths))) self.debug( "profile: started: renderer: %s" % (self.renderer)) try: result = self.render() except: self.error( "%s: exception in rendering" % self ) return ResultBlocks(ResultBlocks( Utils.buildException( "rendering" ) )) finally: self.debug( "profile: finished: renderer: %s" % (self.renderer)) #self.debug( "%s: heap at end\n%s" % (self, str(HP.heap()) )) return result
def buildMatrix( self, work, missing_value = 0, apply_transformations = True, take = None, dtype = numpy.float ): """build a matrix from work, a two-level nested dictionary. If *take* is given, then the matrix will be built from level 3, taking *take* from the deepest level only. This method will also apply conversions if apply_transformations is set. """ labels = DataTree.getPaths( work ) levels = len(labels) if take: if levels != 3: raise ValueError( "expected three labels" ) if take not in labels[-1]: raise ValueError( "no data on `%s`" % take ) take_f = lambda row,column: work[row][column][take] else: if levels != 2: raise ValueError( "expected two levels" ) take_f = lambda row,column: work[row][column] rows, columns = labels[:2] self.debug("creating matrix") matrix = numpy.array( [missing_value] * (len(rows) * len(columns) ), dtype ) matrix.shape = (len(rows), len(columns) ) self.debug("constructing matrix") for x,row in enumerate(rows): for y, column in enumerate(columns): # missing values from DataTree try: v = take_f( row, column ) except KeyError: continue # empty values from DataTree try: if len(v) == 0: continue except TypeError: pass # convert try: matrix[x,y] = v except ValueError: raise ValueError( "malformatted data: expected scalar, got '%s'" % str(work[row][column]) ) except TypeError: raise TypeError( "malformatted data: expected scalar, got '%s'" % str(work[row][column]) ) if self.mConverters and apply_transformations: for converter in self.mConverters: self.debug("applying converter %s" % converter) matrix, rows, columns = converter(matrix, rows, columns) # convert rows/columns to str (might be None) rows = [ str(x) for x in rows ] columns = [ str(x) for x in columns ] return matrix, rows, columns