def __call__(self, data ):
if self.nlevels == None: raise NotImplementedError("incomplete implementation of %s" % str(self))
labels = DataTree.getPaths( data )
debug( "transform: started with paths: %s" % labels)
assert len(labels) >= self.nlevels, "expected at least %i levels - got %i" % (self.nlevels, len(labels))
paths = list(itertools.product( *labels[:-self.nlevels] ))
for path in paths:
work = DataTree.getLeaf( data, path )
if not work: continue
new_data = self.transform( work, path )
if new_data:
if path:
DataTree.setLeaf( data, path, new_data )
else:
# set new root
data = new_data
else:
warn( "no data at %s - removing branch" % str(path))
DataTree.removeLeaf( data, path )
debug( "transform: finished with paths: %s" % DataTree.getPaths( data ))
return data
def __call__(self, *args, **kwargs):
try:
self.parseArguments(*args, **kwargs)
except:
return ResultBlocks(ResultBlocks(Utils.buildException("parsing")))
self.debug("profile: started: tracker: %s" % (self.tracker))
try:
self.collect()
except:
return ResultBlocks(ResultBlocks(Utils.buildException("collection")))
self.debug("profile: finished: tracker: %s" % (self.tracker))
data_paths = DataTree.getPaths(self.data)
self.debug("%s: after collection: %i data_paths: %s" % (self, len(data_paths), str(data_paths)))
# transform data
try:
self.transform()
except:
return ResultBlocks(ResultBlocks(Utils.buildException("transformation")))
data_paths = DataTree.getPaths(self.data)
self.debug("%s: after transformation: %i data_paths: %s" % (self, len(data_paths), str(data_paths)))
# remove superfluous levels
try:
self.prune()
except:
return ResultBlocks(ResultBlocks(Utils.buildException("pruning")))
data_paths = DataTree.getPaths(self.data)
self.debug("%s: after pruning: %i data_paths: %s" % (self, len(data_paths), str(data_paths)))
# remove group plots
try:
self.group()
except:
return ResultBlocks(ResultBlocks(Utils.buildException("grouping")))
data_paths = DataTree.getPaths(self.data)
self.debug("%s: after grouping: %i data_paths: %s" % (self, len(data_paths), str(data_paths)))
self.debug("profile: started: renderer: %s" % (self.renderer))
try:
result = self.render()
except:
return ResultBlocks(ResultBlocks(Utils.buildException("rendering")))
self.debug("profile: finished: renderer: %s" % (self.renderer))
return result
def exclude( self ):
'''exclude data paths.
Only those data paths not matching the exclude term are accepted.
'''
if not self.exclude_paths: return
data_paths = DataTree.getPaths( self.data )
# currently enumerates - bfs more efficient
all_paths = list(itertools.product( *data_paths ))
for path in all_paths:
for s in self.exclude_paths:
if s in path:
self.debug( "%s: ignoring path %s because of :exclude:=%s" % (self.tracker, path, s))
try: DataTree.removeLeaf( self.data, path )
except KeyError: pass
elif s.startswith("r(") and s.endswith(")"):
# collect pattern matches:
# remove r()
s = s[2:-1]
# remove flanking quotation marks
if s[0] in ('"', "'") and s[-1] in ('"', "'"): s = s[1:-1]
rx = re.compile( s )
if any( ( rx.search( p ) for p in path ) ):
self.debug( "%s: ignoring path %s because of :exclude:=%s" % (self.tracker, path, s))
try: DataTree.removeLeaf( self.data, path )
except KeyError: pass
def __call__(self, data, path ):
'''iterate over leaves/branches in data structure.
This method will call the :meth:`render` method for
each leaf/branch at level :attr:`nlevels`.
'''
if self.nlevels == None: raise NotImplementedError("incomplete implementation of %s" % str(self))
result = ResultBlocks( title = path2str(path) )
labels = DataTree.getPaths( data )
if len(labels) < self.nlevels:
self.warn( "at %s: expected at least %i levels - got %i: %s" %\
(str(path), self.nlevels, len(labels), str(labels)) )
result.append( EmptyResultBlock( title = path2str(path) ) )
return result
paths = list(itertools.product( *labels[:-self.nlevels] ))
for p in paths:
work = DataTree.getLeaf( data, p )
if not work: continue
try:
result.extend( self.render( work, path + p ) )
except:
self.warn("exeception raised in rendering for path: %s" % str(path+p))
raise
return result
def render(self):
"""supply the :class:`Renderer.Renderer` with the data to render.
The data supplied will depend on the ``groupby`` option.
return resultblocks
"""
self.debug("%s: rendering data started for %i items" % (self, len(self.data)))
results = ResultBlocks(title="main")
# get number of levels required by renderer
try:
renderer_nlevels = self.renderer.nlevels
except AttributeError:
renderer_nlevels = 0
data_paths = DataTree.getPaths(self.data)
nlevels = len(data_paths)
group_level = self.group_level
self.debug(
"%s: rendering data started. levels=%i, required levels>=%i, group_level=%i, data_paths=%s"
% (self, nlevels, renderer_nlevels, group_level, str(data_paths)[:100])
)
if nlevels < renderer_nlevels:
# add some dummy levels if levels is not enough
d = self.data
for x in range(renderer_nlevels - nlevels):
d = odict((("all", d),))
results.append(self.renderer(d, path=("all",)))
elif group_level < 0 or renderer_nlevels < 0:
# no grouping
results.append(self.renderer(self.data, path=()))
else:
# group at level group_level
paths = list(itertools.product(*data_paths[: group_level + 1]))
for path in paths:
work = DataTree.getLeaf(self.data, path)
if not work:
continue
try:
results.append(self.renderer(work, path=path))
except:
results.append(ResultBlocks(Utils.buildException("rendering")))
if len(results) == 0:
self.warn("tracker returned no data.")
raise ValueError("tracker returned no data.")
self.debug("%s: rendering data finished with %i blocks" % (self.tracker, len(results)))
return results
def prune(self):
"""prune data tree.
Remove all empty leaves.
Remove all levels from the data tree that are
superfluous, i.e. levels that contain only a single label
all labels in the hierarchy below are the same.
Ignore both the first and last level for this analyis.
"""
# remove all empty leaves
DataTree.removeEmptyLeaves(self.data)
# prune superfluous levels
data_paths = DataTree.getPaths(self.data)
nlevels = len(data_paths)
# get number of levels required by renderer
try:
renderer_nlevels = self.renderer.nlevels
except AttributeError:
renderer_nlevels = 0
# do not prune for renderers that want all data
if renderer_nlevels < 0:
return
levels_to_prune = []
for level in range(1, nlevels - 1):
# check for single label in level
if len(data_paths[level]) == 1:
label = data_paths[level][0]
prefixes = DataTree.getPrefixes(self.data, level)
keep = False
for prefix in prefixes:
leaves = DataTree.getLeaf(self.data, prefix)
if len(leaves) > 1 or label not in leaves:
keep = True
break
if not keep:
levels_to_prune.append((level, label))
levels_to_prune.reverse()
# only prune to the minimum of levels required by renderer at most
# levels_to_prune = levels_to_prune[:nlevels - renderer_nlevels]
for level, label in levels_to_prune:
self.debug("pruning level %i from data tree: label='%s'" % (level, label))
DataTree.removeLevel(self.data, level)
def group( self ):
'''rearrange data tree for grouping
and set group level.
Through grouping the data tree is rearranged such
that the level at which data will be grouped will
be the top (0th) level in the nested dictionary.
'''
data_paths = DataTree.getPaths( self.data )
nlevels = len(data_paths)
# get number of levels required by renderer
try:
renderer_nlevels = self.renderer.nlevels
except AttributeError:
renderer_nlevels = 0
if self.groupby == "none":
self.group_level = nlevels - 1
elif self.groupby == "track":
# track is first level
self.group_level = 1
# add pseudo levels, if there are not enough levels
# to group by track
if nlevels == renderer_nlevels:
d = odict()
for x in data_paths[0]: d[x] = odict( ((x, self.data[x]),))
self.data = d
elif self.groupby == "slice":
# rearrange tracks and slices in data tree
if nlevels <= 2 :
self.warn( "grouping by slice, but only %i levels in data tree - all are grouped" % nlevels)
self.group_level = 0
else:
self.data = DataTree.swop( self.data, 0, 1)
self.group_level = 1
elif self.groupby == "all":
# group everything together
self.group_level = 0
else:
# neither group by slice or track ("ungrouped")
self.group_level = 0
return self.data
def render( self, work, path ):
# initiate output structure
results = ResultBlocks( title = path2str(path) )
labels = DataTree.getPaths( work )
# iterate over all items at leaf
for path, branch in DataTree.getNodes( work, len(labels) - 2 ):
for key in Utils.TrackerKeywords:
if key in branch:
# add a result block
results.append( ResultBlock( branch[key],
title = path2str(path) ) )
return results
def transform(self,data,path):
from rpy2.robjects import r as R
paths, lengths, values = [],[],[]
labels = DataTree.getPaths(data)
paths = list(itertools.product( *labels[:-1]))
for path in paths:
work = DataTree.getLeaf(data, path)
try:
lengths.append(len(work[self.pval]))
values.extend(work[self.pval])
except TypeError:
lengths.append(0)
values.append(work[self.pval])
padj = R["p.adjust"](values, method = self.method)
padj = [x for x in padj]
for path in paths:
num = lengths.pop(0)
if num > 0:
new_values = padj[0:num]
padj = padj[num:]
else:
new_values = padj[0]
padj = padj[1:]
if path:
work = odict(DataTree.getLeaf(data,path))
work["P-adjust"] = new_values
DataTree.setLeaf(data,path,work)
else:
data["P-adjust"] = new_values
return data
def group(self):
"""rearrange data tree for grouping.
and set group level.
"""
data_paths = DataTree.getPaths(self.data)
nlevels = len(data_paths)
# get number of levels required by renderer
try:
renderer_nlevels = self.renderer.nlevels
except AttributeError:
renderer_nlevels = 0
if self.groupby == "none":
self.group_level = renderer_nlevels
elif self.groupby == "track":
# track is first level
self.group_level = 0
# add pseudo levels, if there are not enough levels
# to group by track
if nlevels == renderer_nlevels:
d = odict()
for x in data_paths[0]:
d[x] = odict(((x, self.data[x]),))
self.data = d
elif self.groupby == "slice":
# rearrange tracks and slices in data tree
if nlevels <= 2:
warn("grouping by slice, but only %i levels in data tree - all are grouped" % nlevels)
self.group_level = -1
else:
self.data = DataTree.swop(self.data, 0, 1)
self.group_level = 0
elif self.groupby == "all":
# group everthing together
self.group_level = -1
else:
# neither group by slice or track ("ungrouped")
self.group_level = -1
def __call__ (self, data):
if self.nlevels == 0:
self.nlevels = len(DataTree.getPaths(data))
return Transformer.__call__(self,data)
def __call__(self, *args, **kwargs ):
#self.debug( "%s: heap at start\n%s" % (self, str(HP.heap()) ))
try: self.parseArguments( *args, **kwargs )
except:
self.error( "%s: exception in parsing" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "parsing" ) ))
# collect no data if tracker is the empty tracker
# and go straight to rendering
try:
if self.tracker.getTracks() == ["empty"]:
# is instance does not work because of module mapping
# type(Tracker.Empty) == SphinxReport.Tracker.Empty
# type(self.tracker) == Tracker.Empty
# if isinstance( self.tracker, Tracker.Empty):
result =self.renderer()
return ResultBlocks( result )
except AttributeError:
# for function trackers
pass
self.debug( "profile: started: tracker: %s" % (self.tracker))
# collecting data
try: self.collect()
except:
self.error( "%s: exception in collection" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "collection" ) ))
finally:
self.debug( "profile: finished: tracker: %s" % (self.tracker))
if len(self.data) == 0:
self.info( "%s: no data - processing complete" % self.tracker )
return None
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after collection: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# self.debug( "%s: heap after collection\n%s" % (self, str(HP.heap()) ))
# transform data
try: self.transform()
except:
self.error( "%s: exception in transformation" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "transformation" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after transformation: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# special Renderers - do not proceed
# Special renderers
if isinstance( self.renderer, Renderer.User):
results = ResultBlocks( title="main" )
results.append( self.renderer( self.data, ('') ) )
return results
elif isinstance( self.renderer, Renderer.Debug):
results = ResultBlocks( title="main" )
results.append( self.renderer( self.data, ('') ) )
return results
# self.debug( "%s: heap after transformation\n%s" % (self, str(HP.heap()) ))
# restrict
try: self.restrict()
except:
self.error( "%s: exception in restrict" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "restrict" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after restrict: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# exclude
try: self.exclude()
except:
self.error( "%s: exception in exclude" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "exclude" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after exclude: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# remove superfluous levels
try: self.prune()
except:
self.error( "%s: exception in pruning" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "pruning" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after pruning: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# remove group plots
try: self.group()
except:
self.error( "%s: exception in grouping" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "grouping" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after grouping: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
self.debug( "profile: started: renderer: %s" % (self.renderer))
try: result = self.render()
except:
self.error( "%s: exception in rendering" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "rendering" ) ))
finally:
self.debug( "profile: finished: renderer: %s" % (self.renderer))
#self.debug( "%s: heap at end\n%s" % (self, str(HP.heap()) ))
return result
def buildMatrix( self,
work,
missing_value = 0,
apply_transformations = True,
take = None,
dtype = numpy.float ):
"""build a matrix from work, a two-level nested dictionary.
If *take* is given, then the matrix will be built from
level 3, taking *take* from the deepest level only.
This method will also apply conversions if apply_transformations
is set.
"""
labels = DataTree.getPaths( work )
levels = len(labels)
if take:
if levels != 3: raise ValueError( "expected three labels" )
if take not in labels[-1]: raise ValueError( "no data on `%s`" % take )
take_f = lambda row,column: work[row][column][take]
else:
if levels != 2: raise ValueError( "expected two levels" )
take_f = lambda row,column: work[row][column]
rows, columns = labels[:2]
self.debug("creating matrix")
matrix = numpy.array( [missing_value] * (len(rows) * len(columns) ), dtype )
matrix.shape = (len(rows), len(columns) )
self.debug("constructing matrix")
for x,row in enumerate(rows):
for y, column in enumerate(columns):
# missing values from DataTree
try:
v = take_f( row, column )
except KeyError:
continue
# empty values from DataTree
try:
if len(v) == 0: continue
except TypeError:
pass
# convert
try:
matrix[x,y] = v
except ValueError:
raise ValueError( "malformatted data: expected scalar, got '%s'" % str(work[row][column]) )
except TypeError:
raise TypeError( "malformatted data: expected scalar, got '%s'" % str(work[row][column]) )
if self.mConverters and apply_transformations:
for converter in self.mConverters:
self.debug("applying converter %s" % converter)
matrix, rows, columns = converter(matrix, rows, columns)
# convert rows/columns to str (might be None)
rows = [ str(x) for x in rows ]
columns = [ str(x) for x in columns ]
return matrix, rows, columns
