def render( self, work, path ):
# initiate output structure
results = ResultBlocks( title = path )
# iterate over all items at leaf
for key in work:
t = type(work[key])
try:
l = "%i" % len(work[key])
except AttributeError:
l = "na"
except TypeError:
l = "na"
# add a result block.
data = str(work[key])
if len(data) > 30: data=data[:30] + "..."
results.append( ResultBlock( "path= %s, type= %s, len= %s, data= %s" % \
( path2str(path + (key,)),
t, l, data),
title = "") )
return results
def render(self, dataframe, path ):
if len(dataframe.columns) < 2:
raise ValueError( "requiring two coordinates, only got %s" % str(dataframe.columns))
plts, legend = [], []
blocks = ResultBlocks()
for xcolumn, ycolumn in itertools.combinations( dataframe.columns, 2 ):
# remove missing data points
xvalues, yvalues = Stats.filterMissing( (dataframe[xcolumn], dataframe[ycolumn]) )
# remove columns with all NaN
if len(xvalues) == 0 or len(yvalues) == 0:
continue
# apply log transformation on data not on plot
if self.logscale:
if "x" in self.logscale:
xvalues = R.log10(xvalues)
if "y" in self.logscale:
yvalues = R.log10(yvalues)
self.startPlot()
# wrap, as pandas series can not
# passed through rpy2.
R.smoothScatter( numpy.array( xvalues, dtype=numpy.float),
numpy.array( yvalues, dtype=numpy.float),
xlab=xcolumn,
ylab=ycolumn,
nbin = self.nbins )
blocks.extend( self.endPlot( dataframe, path ) )
return blocks
def __call__(self, data, path ):
'''iterate over leaves/branches in data structure.
This method will call the :meth:`render` method for
each leaf/branch at level :attr:`nlevels`.
'''
if self.nlevels == None: raise NotImplementedError("incomplete implementation of %s" % str(self))
result = ResultBlocks( title = path2str(path) )
labels = DataTree.getPaths( data )
if len(labels) < self.nlevels:
self.warn( "at %s: expected at least %i levels - got %i: %s" %\
(str(path), self.nlevels, len(labels), str(labels)) )
result.append( EmptyResultBlock( title = path2str(path) ) )
return result
paths = list(itertools.product( *labels[:-self.nlevels] ))
for p in paths:
work = DataTree.getLeaf( data, p )
if not work: continue
try:
result.extend( self.render( work, path + p ) )
except:
self.warn("exeception raised in rendering for path: %s" % str(path+p))
raise
return result
def __call__(self, track, slice=None):
# note there are spaces behind the %(image)s directive to accomodate
# for path substitution
block = '''
.. figure:: %(image)s
:height: 300
'''
blocks = ResultBlocks()
tracks = sorted([x.asFile() for x in TRACKS])
for track in tracks:
files = glob.glob(
os.path.join(EXPORTDIR, "fastqc", "%s*_fastqc" % track))
for x, fn in enumerate(sorted(files)):
y = x + 1
image = os.path.abspath(
os.path.join(fn, "Images", "%s.png" % slice))
if not os.path.exists(image):
continue
blocks.append(ResultBlock(text=block % locals(),
title=os.path.basename(fn)))
return odict((("rst", "\n".join(Utils.layoutBlocks(blocks, layout="columns-2"))),))
def render( self, work, path ):
"""render the data.
"""
results = ResultBlocks( title = path )
matrix, rows, columns = self.buildMatrix( work )
title = path2str(path)
if len(rows) == 0:
return ResultBlocks( ResultBlock( "", title = title) )
# do not output large matrices as rst files
if not self.force and (len(rows) > self.max_rows or len(columns) > self.max_cols):
return self.asFile( [ [ self.toString(x) for x in r ] for r in matrix ],
rows,
columns,
title )
lines = []
lines.append( ".. csv-table:: %s" % title )
lines.append( ' :header: "track","%s" ' % '","'.join( columns ) )
lines.append( '')
for x in range(len(rows)):
lines.append( ' "%s","%s"' % (rows[x], '","'.join( [ self.toString(x) for x in matrix[x,:] ] ) ) )
lines.append( "")
if not path: subtitle = ""
else: subtitle = path2str(path)
results.append( ResultBlock( "\n".join(lines), title = subtitle ) )
return results
def __call__(self, track, slice=None):
# note there are spaces behind the %(image)s directive to accomodate
# for path substitution
block = '''
.. figure:: %(image)s
:height: 300
'''
blocks = ResultBlocks()
tracks = sorted([x.asFile() for x in TRACKS])
for track in tracks:
files = glob.glob(
os.path.join(EXPORTDIR, "fastqc", "%s*_fastqc" % track))
for x, fn in enumerate(sorted(files)):
y = x + 1
image = os.path.abspath(
os.path.join(fn, "Images", "%s.png" % slice))
if not os.path.exists(image):
continue
blocks.append(
ResultBlock(text=block % locals(),
title=os.path.basename(fn)))
return odict(
(("rst", "\n".join(Utils.layoutBlocks(blocks,
layout="columns-2"))), ))
def __call__(self, track, slice = None ):
edir = EXPORTDIR
method = "utr_extension"
blocks = ResultBlocks()
filepath = "%(edir)s/%(method)s/%(track)s.readextension_%(region)s_%(direction)s.%(slice)s.png"
block = \
'''
.. figure:: %(filename)s
:height: 300
'''
# append spaces for file extension
block = "\n".join( [ x + " " * 200 for x in block.split("\n") ] )
for region, direction in itertools.product( ("downstream", "upstream"),
("sense", "antisense", "anysense" )):
filename = filepath % locals()
if os.path.exists( filename ):
blocks.append( ResultBlock( text = block % locals(),
title = "%(track)s %(region)s %(direction)s" % locals() ) )
# else:
# blocks.append( ResultBlock( "",
# title = "%(track)s %(region)s %(direction)s" % locals() ) )
return odict( (("rst", "\n".join(Utils.layoutBlocks( blocks, layout="columns-3" ))),))
def __call__(self, track, slice = None ):
edir = EXPORTDIR
method = "utr_extension"
blocks = ResultBlocks()
filepath = "%(edir)s/%(method)s/%(track)s.readextension_%(region)s_%(direction)s.%(slice)s.png"
block = \
'''
.. figure:: %(filename)s
:height: 300
'''
# append spaces for file extension
block = "\n".join( [ x + " " * 200 for x in block.split("\n") ] )
for region, direction in itertools.product( ("downstream", "upstream"),
("sense", "antisense", "anysense" )):
filename = filepath % locals()
if os.path.exists( filename ):
blocks.append( ResultBlock( text = block % locals(),
title = "%(track)s %(region)s %(direction)s" % locals() ) )
# else:
# blocks.append( ResultBlock( "",
# title = "%(track)s %(region)s %(direction)s" % locals() ) )
return odict( (("rst", "\n".join(Utils.layoutBlocks( blocks, layout="columns-3" ))),))
def render( self, data, path ):
# initiate output structure
results = ResultBlocks( title = path )
try:
results.append( ResultBlock(json.dumps(data, indent=4), title=''))
except TypeError:
results.append( ResultBlock(str(data), title=''))
return results
def __call__(self, track, slice = None ):
blocks = ResultBlocks()
block = '''
.. figure:: %(image)s
:height: 300
'''
for image in glob.glob( os.path.join( IMAGEDIR, "*.png" )):
blocks.append( ResultBlock( text = block % locals(),
title = "image" ) )
return odict( (("rst", "\n".join( Utils.layoutBlocks( blocks, layout = "columns-2"))),))
def render(self):
"""supply the :class:`Renderer.Renderer` with the data to render.
The data supplied will depend on the ``groupby`` option.
return resultblocks
"""
self.debug("%s: rendering data started for %i items" % (self, len(self.data)))
results = ResultBlocks(title="main")
# get number of levels required by renderer
try:
renderer_nlevels = self.renderer.nlevels
except AttributeError:
renderer_nlevels = 0
data_paths = DataTree.getPaths(self.data)
nlevels = len(data_paths)
group_level = self.group_level
self.debug(
"%s: rendering data started. levels=%i, required levels>=%i, group_level=%i, data_paths=%s"
% (self, nlevels, renderer_nlevels, group_level, str(data_paths)[:100])
)
if nlevels < renderer_nlevels:
# add some dummy levels if levels is not enough
d = self.data
for x in range(renderer_nlevels - nlevels):
d = odict((("all", d),))
results.append(self.renderer(d, path=("all",)))
elif group_level < 0 or renderer_nlevels < 0:
# no grouping
results.append(self.renderer(self.data, path=()))
else:
# group at level group_level
paths = list(itertools.product(*data_paths[: group_level + 1]))
for path in paths:
work = DataTree.getLeaf(self.data, path)
if not work:
continue
try:
results.append(self.renderer(work, path=path))
except:
results.append(ResultBlocks(Utils.buildException("rendering")))
if len(results) == 0:
self.warn("tracker returned no data.")
raise ValueError("tracker returned no data.")
self.debug("%s: rendering data finished with %i blocks" % (self.tracker, len(results)))
return results
def render( self, work, path ):
# initiate output structure
results = ResultBlocks( title = path2str(path) )
labels = DataTree.getPaths( work )
# iterate over all items at leaf
for path, branch in DataTree.getNodes( work, len(labels) - 2 ):
for key in Utils.TrackerKeywords:
if key in branch:
# add a result block
results.append( ResultBlock( branch[key],
title = path2str(path) ) )
return results
def __call__(self, dataframe, path ):
'''iterate over leaves/branches in data structure.
This method will call the :meth:`render` method for
each leaf/branch at level :attr:`nlevels`.
'''
if self.nlevels == None: raise NotImplementedError("incomplete implementation of %s" % str(self))
try:
labels = dataframe.index.levels
paths = dataframe.index.unique()
except AttributeError:
labels = ['dummy1']
paths = ['dummy1']
result = ResultBlocks()
#print len(labels), self.nlevels
#print 'dataframe=', dataframe
if self.nlevels != -1 and len(labels) != self.nlevels:
raise ValueError( "at path %s: expected %i levels - got %i: %s" %\
(str(path), self.nlevels,
len(labels), str(labels)) )
#result.append( EmptyResultBlock( title = path2str(path) ) )
#return result
if not self.split_at:
# print without splitting
result.extend( self.render( dataframe, path ) )
else:
# split dataframe at first index
first_level_labels = dataframe.index.get_level_values(0).unique()
if len(first_level_labels) < self.split_at:
result.extend( self.render( dataframe, path ) )
else:
# select tracks to always add to split
# pick always tracks
if self.split_always:
always = [ x for x, y in itertools.product( first_level_labels, self.split_always) \
if re.search( y, x ) ]
else:
always = []
for z, x in enumerate(range( 0, len(first_level_labels), self.split_at)) :
select = list(DataTree.unique( always + list(first_level_labels[x:x+self.split_at]) ))
if len(dataframe.index.names) == 1:
# if only one level, use loc to obtain dataframe
# index is duplicated, so ignore second level
work = pandas.concat( [dataframe.loc[[s]] for s in select], keys = select )
work.reset_index( range( 1, len(work.index.names)), drop=True, inplace=True )
else:
work = pandas.concat( [dataframe.xs(s, axis=0) for s in select], keys = select )
# reconcile index names
work.index.names = dataframe.index.names
result.extend( self.render( work, path + (z, ) ) )
return result
def render( self, work, path ):
# initiate output structure
results = ResultBlocks( title = path )
# iterate over all items at leaf
#for key in work:
if "text" in work:
# add a result block
results.append( ResultBlock( work["text"],
title = "" ) )
elif "rst" in work:
# add a result block
results.append( ResultBlock( work["rst"],
title = "" ) )
return results
def __call__(self, dataframe, path):
# modify table (adding/removing columns) according to user options
# matrix, row_headers, col_headers = self.modifyTable( matrix, row_headers, col_headers )
dataframe = self.modifyTable( dataframe )
title = path2str(path)
results = ResultBlocks()
row_headers = dataframe.index
col_headers = dataframe.columns
# do not output large matrices as rst files
if self.separate or (not self.force and
(len(row_headers) > self.max_rows or
len(col_headers) > self.max_cols)):
if self.large == "xls":
results.append( self.asSpreadSheet( dataframe, row_headers, col_headers, title ) )
else:
results.append( self.asFile( dataframe, row_headers, col_headers, title ) )
if self.preview:
row_headers = row_headers[:self.max_rows]
col_headers = col_headers[:self.max_cols]
matrix = [ x[:self.max_cols] for x in matrix[:self.max_rows] ]
else:
return results
out = StringIO.StringIO()
dataframe.to_csv( out )
lines = []
lines.append( ".. csv-table:: %s" % title )
lines.append( " :class: sortable" )
if self.add_rowindex:
lines.append( ' :header: "row", "", "%s" ' % '","'.join( map(str, col_headers) ) )
lines.append( '' )
x = 0
for header, line in zip( row_headers, matrix ):
x += 1
lines.append( ' %i,"%s","%s"' % (x, str(header), '","'.join( map(str, line) ) ) )
else:
l = out.getvalue().split("\n")
lines.append( ' :header: %s' % l[0] )
lines.append( '' )
lines.extend( [' %s' % x for x in l[1:] ] )
lines.append( "")
results.append( ResultBlock( "\n".join(lines), title = title) )
return results
def render( self, work, path ):
"""render the data.
"""
results = ResultBlocks( title = path )
matrix, rows, columns = self.buildMatrix( work )
title = path2str(path)
if len(rows) == 0:
return ResultBlocks( ResultBlock( "", title = title) )
# do not output large matrices as rst files
# separate and force need to be mixed in.
if self.separate or (not self.force and (len(rows) > self.max_rows or len(columns) > self.max_cols)):
return ResultBlocks( self.asFile( pandas.DataFrame( matrix,
index = rows,
columns = columns),
rows,
columns,
title ),
title = path )
lines = []
lines.append( ".. csv-table:: %s" % title )
lines.append( " :class: sortable" )
lines.append( ' :header: "track","%s" ' % '","'.join( columns ) )
lines.append( '')
for x in range(len(rows)):
lines.append( ' "%s","%s"' % (rows[x], '","'.join( [ self.toString(x) for x in matrix[x,:] ] ) ) )
lines.append( "")
if path is None: subtitle = ""
else: subtitle = path2str(path)
results.append( ResultBlock( "\n".join(lines), title = subtitle ) )
return results
########################################################
# create and call dispatcher
logging.debug( "report_directive.run: creating dispatcher" )
dispatcher = Dispatcher.Dispatcher( tracker,
renderer,
transformers )
blocks = dispatcher( **dispatcher_options )
except:
logging.warn("report_directive.run: exception caught at %s:%i - see document" % (str(document), lineno) )
blocks = ResultBlocks(ResultBlocks( Utils.buildException( "invocation" ) ))
code = None
tracker_id = None
########################################################
## write code output
linked_codename = re.sub( "\\\\", "/", os.path.join( rst2srcdir, codename ))
if code and basedir != outdir:
outfile = open( os.path.join(outdir, codename ), "w" )
for line in code: outfile.write( line )
outfile.close()
###########################################################
# collect images
###########################################################
map_figure2text = {}
def layoutBlocks( blocks, layout = "column"):
"""layout blocks of rst text.
layout can be one of "column", "row", or "grid".
The layout uses an rst table to arrange elements.
"""
lines = []
if len(blocks) == 0: return lines
# flatten blocks
x = ResultBlocks()
for b in blocks:
if b.title: b.updateTitle( b.title, "prefix" )
try:
x.extend( b )
except TypeError:
x.append( b )
blocks = x
if layout == "column":
for block in blocks:
if block.title:
lines.extend( block.title.split("\n") )
lines.append( "" )
else:
logging.warn( "report_directive.layoutBlocks: missing title" )
lines.extend(block.text.split("\n"))
lines.extend( [ "", ] )
return lines
elif layout in ("row", "grid"):
if layout == "row": ncols = len(blocks)
elif layout == "grid": ncols = int(math.ceil(math.sqrt( len(blocks) )))
elif layout.startswith("column"):
ncols = min( len(blocks), int(layout.split("-")[1]))
# TODO: think about appropriate fix for empty data
if ncols == 0:
ncols = 1
return lines
else:
raise ValueError( "unknown layout %s " % layout )
# compute column widths
widths = [ x.getWidth() for x in blocks ]
text_heights = [ x.getTextHeight() for x in blocks ]
title_heights = [ x.getTitleHeight() for x in blocks ]
columnwidths = []
for x in range(ncols):
columnwidths.append( max( [widths[y] for y in range( x, len(blocks), ncols ) ] ) )
separator = "+%s+" % "+".join( ["-" * x for x in columnwidths ] )
## add empty blocks
if len(blocks) % ncols:
blocks.extend( [ ResultBlock( "", "" )] * (ncols - len(blocks) % ncols) )
for nblock in range(0, len(blocks), ncols ):
## add text
lines.append( separator )
max_height = max( text_heights[nblock:nblock+ncols] )
new_blocks = ResultBlocks()
for xx in range(nblock, min(nblock+ncols,len(blocks))):
txt, col = blocks[xx].text.split("\n"), xx % ncols
max_width = columnwidths[col]
# add missig lines
txt.extend( [""] * (max_height - len(txt)) )
# extend lines
txt = [ x + " " * (max_width - len(x)) for x in txt ]
new_blocks.append( txt )
for l in zip( *new_blocks ):
lines.append( "|%s|" % "|".join( l ) )
## add subtitles
max_height = max( title_heights[nblock:nblock+ncols] )
if max_height > 0:
new_blocks = ResultBlocks()
lines.append( separator )
for xx in range(nblock, min(nblock+ncols,len(blocks))):
txt, col = blocks[xx].title.split("\n"), xx % ncols
max_width = columnwidths[col]
# add missig lines
txt.extend( [""] * (max_height - len(txt) ) )
# extend lines
txt = [ x + " " * (max_width - len(x)) for x in txt ]
new_blocks.append( txt )
for l in zip( *new_blocks ):
lines.append( "|%s|" % "|".join( l ) )
lines.append( separator )
lines.append( "" )
return lines
def run(arguments, options, lineno, content, state_machine=None, document=None, srcdir=None, builddir=None):
"""process :report: directive.
*srdir* - top level directory of rst documents
*builddir* - build directory
"""
tag = "%s:%i" % (str(document), lineno)
logging.debug("report_directive.run: profile: started: rst: %s" % tag)
# sort out the paths
# reference is used for time-stamping
tracker_name = directives.uri(arguments[0])
basedir, fname, basename, ext, outdir, codename, notebookname = Utils.buildPaths(tracker_name)
# get the directory of the rst file
rstdir, rstfile = os.path.split(document) # state_machine.document.attributes['source'])
# root of document tree
if srcdir is None:
srcdir = setup.srcdir
# build directory
if builddir is None:
builddir = setup.confdir
# remove symbolic links
srcdir, builddir, rstdir = [os.path.realpath(x) for x in (srcdir, builddir, rstdir)]
# there are three directories:
# builddir = directory where document is built in (usually _build/html or similar)
# rstdir = directory where rst sources are located
# srcdir = directory from which the build process is started
# path to root relative to rst
rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir)
# path to root relative to rst
rst2builddir = os.path.join(os.path.relpath(builddir, start=rstdir), outdir)
# path relative to source (for images)
root2builddir = os.path.join(os.path.relpath(builddir, start=srcdir), outdir)
logging.debug(
"report_directive.run: arguments=%s, options=%s, lineno=%s, content=%s, document=%s"
% (str(arguments), str(options), str(lineno), str(content), str(document))
)
logging.debug(
"report_directive.run: plotdir=%s, basename=%s, ext=%s, fname=%s, rstdir=%s, srcdir=%s, builddir=%s"
% (tracker_name, basename, ext, fname, rstdir, srcdir, builddir)
)
logging.debug(
"report_directive.run: tracker_name=%s, basedir=%s, rst2src=%s, root2build=%s, outdir=%s, codename=%s"
% (tracker_name, basedir, rst2srcdir, rst2builddir, outdir, codename)
)
# try to create. If several processes try to create it,
# testing with `if` will not work.
try:
os.makedirs(outdir)
except OSError as msg:
pass
if not os.path.exists(outdir):
raise OSError("could not create directory %s: %s" % (outdir, msg))
########################################################
# collect options
# replace placedholders
try:
options = Utils.updateOptions(options)
except ValueError as msg:
logging.warn("failure while updating options: %s" % msg)
logging.debug("report_directive.run: options=%s" % (str(options),))
transformer_names = []
renderer_name = None
# get layout option
layout = options.get("layout", "column")
option_map = getOptionMap()
renderer_options = Utils.selectAndDeleteOptions(options, option_map["render"])
transformer_options = Utils.selectAndDeleteOptions(options, option_map["transform"])
dispatcher_options = Utils.selectAndDeleteOptions(options, option_map["dispatch"])
tracker_options = Utils.selectAndDeleteOptions(options, option_map["tracker"])
display_options = Utils.selectAndDeleteOptions(options, option_map["display"])
logging.debug("report_directive.run: renderer options: %s" % str(renderer_options))
logging.debug("report_directive.run: transformer options: %s" % str(transformer_options))
logging.debug("report_directive.run: dispatcher options: %s" % str(dispatcher_options))
logging.debug("report_directive.run: tracker options: %s" % str(tracker_options))
logging.debug("report_directive.run: display options: %s" % str(display_options))
if "transform" in display_options:
transformer_names = display_options["transform"].split(",")
del display_options["transform"]
if "render" in display_options:
renderer_name = display_options["render"]
del display_options["render"]
########################################################
# check for missing files
if renderer_name != None:
options_key = (
str(renderer_options)
+ str(transformer_options)
+ str(dispatcher_options)
+ str(tracker_options)
+ str(transformer_names)
)
options_hash = hashlib.md5(options_key.encode()).hexdigest()
template_name = Utils.quote_filename(Config.SEPARATOR.join((tracker_name, renderer_name, options_hash)))
filename_text = os.path.join(outdir, "%s.txt" % (template_name))
logging.debug("report_directive.run: options_hash=%s" % options_hash)
###########################################################
# check for existing files
# update strategy does not use file stamps, but checks
# for presence/absence of text element and if all figures
# mentioned in the text element are present
###########################################################
queries = [re.compile("%s(%s\S+.%s)" % (root2builddir, outdir, suffix)) for suffix in ("png", "pdf")]
logging.debug("report_directive.run: checking for changed files.")
# check if text element exists
if os.path.exists(filename_text):
lines = [x[:-1] for x in open(filename_text, "r").readlines()]
filenames = []
# check if all figures are present
for line in lines:
for query in queries:
x = query.search(line)
if x:
filenames.extend(list(x.groups()))
logging.debug("report_directive.run: %s: checking for %s" % (tag, str(filenames)))
for filename in filenames:
if not os.path.exists(filename):
logging.info("report_directive.run: %s: redo: %s missing" % (tag, filename))
break
else:
logging.info("report_directive.run: %s: noredo: all files are present" % tag)
## all is present - save text and return
if lines and state_machine:
state_machine.insert_input(lines, state_machine.input_lines.source(0))
return []
else:
logging.debug("report_directive.run: %s: no check performed: %s missing" % (tag, str(filename_text)))
else:
template_name = ""
filename_text = None
##########################################################
# Initialize collectors
collectors = []
for collector in list(getPlugins("collect").values()):
collectors.append(collector())
##########################################################
## instantiate tracker, dispatcher, renderer and transformers
## and collect output
###########################################################
try:
########################################################
# find the tracker
logging.debug("report_directive.run: collecting tracker %s with options %s " % (tracker_name, tracker_options))
code, tracker = Utils.makeTracker(tracker_name, (), tracker_options)
if not tracker:
logging.error("report_directive.run: no tracker - no output from %s " % str(document))
raise ValueError("tracker `%s` not found" % tracker_name)
logging.debug("report_directive.run: collected tracker %s" % tracker_name)
tracker_id = Cache.tracker2key(tracker)
########################################################
# determine the transformer
logging.debug("report_directive.run: creating transformers")
transformers = Utils.getTransformers(transformer_names, transformer_options)
########################################################
# determine the renderer
logging.debug("report_directive.run: creating renderer.")
if renderer_name == None:
logging.error("report_directive.run: no renderer - no output from %s" % str(document))
raise ValueError("the report directive requires a renderer")
renderer = Utils.getRenderer(renderer_name, renderer_options)
########################################################
# create and call dispatcher
logging.debug("report_directive.run: creating dispatcher")
dispatcher = Dispatcher.Dispatcher(tracker, renderer, transformers)
# add the tracker options
dispatcher_options.update(tracker_options)
blocks = dispatcher(**dispatcher_options)
if blocks == None:
blocks = ResultBlocks(
ResultBlocks(Utils.buildWarning("NoData", "tracker %s returned no Data" % str(tracker)))
)
code = None
tracker_id = None
except:
logging.warn("report_directive.run: exception caught at %s:%i - see document" % (str(document), lineno))
blocks = ResultBlocks(ResultBlocks(Utils.buildException("invocation")))
code = None
tracker_id = None
logging.debug("report_directive.run: profile: started: collecting: %s" % tag)
########################################################
## write code output
linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename))
if code and basedir != outdir:
with open(os.path.join(outdir, codename), "w") as outfile:
for line in code:
outfile.write(line)
########################################################
## write notebook snippet
linked_notebookname = re.sub("\\\\", "/", os.path.join(rst2srcdir, notebookname))
if basedir != outdir:
with open(os.path.join(outdir, notebookname), "w") as outfile:
Utils.writeNoteBookEntry(
outfile,
renderer=renderer_name,
tracker=tracker_name,
transformers=transformer_names,
options=renderer_options.items() + tracker_options.items() + transformer_options.items(),
)
###########################################################
# collect images
###########################################################
map_figure2text = {}
try:
for collector in collectors:
map_figure2text.update(
collector.collect(
blocks,
template_name,
outdir,
rstdir,
builddir,
srcdir,
content,
display_options,
tracker_id,
links={"code_url": linked_codename, "notebook_url": linked_notebookname},
)
)
except:
logging.warn(
"report_directive.run: exception caught while collecting with %s at %s:%i - see document"
% (collector, str(document), lineno)
)
blocks = ResultBlocks(ResultBlocks(Utils.buildException("collection")))
code = None
tracker_id = None
###########################################################
# replace place holders or add text
###########################################################
## add default for text-only output
map_figure2text["default-prefix"] = TEMPLATE_TEXT % locals()
map_figure2text["default-suffix"] = ""
blocks.updatePlaceholders(map_figure2text)
###########################################################
## render the output taking into account the layout
lines = Utils.layoutBlocks(blocks, layout)
###########################################################
# add caption
lines.extend(["::", ""])
if content:
lines.extend([" %s" % row.strip() for row in content])
lines.append("")
lines.append("")
# output rst text for this renderer
if filename_text:
outfile = open(filename_text, "w")
outfile.write("\n".join(lines))
outfile.close()
if SPHINXREPORT_DEBUG:
for x, l in enumerate(lines):
print("%5i %s" % (x, l))
if len(lines) and state_machine:
state_machine.insert_input(lines, state_machine.input_lines.source(0))
logging.debug("report_directive.run: profile: finished: collecting: %s" % tag)
logging.debug("report_directive.run: profile: finished: rst: %s:%i" % (str(document), lineno))
return []
def __call__(self, *args, **kwargs ):
#self.debug( "%s: heap at start\n%s" % (self, str(HP.heap()) ))
try: self.parseArguments( *args, **kwargs )
except:
self.error( "%s: exception in parsing" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "parsing" ) ))
# collect no data if tracker is the empty tracker
# and go straight to rendering
try:
if self.tracker.getTracks() == ["empty"]:
# is instance does not work because of module mapping
# type(Tracker.Empty) == SphinxReport.Tracker.Empty
# type(self.tracker) == Tracker.Empty
# if isinstance( self.tracker, Tracker.Empty):
result =self.renderer()
return ResultBlocks( result )
except AttributeError:
# for function trackers
pass
self.debug( "profile: started: tracker: %s" % (self.tracker))
# collecting data
try: self.collect()
except:
self.error( "%s: exception in collection" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "collection" ) ))
finally:
self.debug( "profile: finished: tracker: %s" % (self.tracker))
if len(self.data) == 0:
self.info( "%s: no data - processing complete" % self.tracker )
return None
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after collection: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# self.debug( "%s: heap after collection\n%s" % (self, str(HP.heap()) ))
# transform data
try: self.transform()
except:
self.error( "%s: exception in transformation" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "transformation" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after transformation: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# special Renderers - do not proceed
# Special renderers
if isinstance( self.renderer, Renderer.User):
results = ResultBlocks( title="main" )
results.append( self.renderer( self.data, ('') ) )
return results
elif isinstance( self.renderer, Renderer.Debug):
results = ResultBlocks( title="main" )
results.append( self.renderer( self.data, ('') ) )
return results
# self.debug( "%s: heap after transformation\n%s" % (self, str(HP.heap()) ))
# restrict
try: self.restrict()
except:
self.error( "%s: exception in restrict" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "restrict" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after restrict: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# exclude
try: self.exclude()
except:
self.error( "%s: exception in exclude" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "exclude" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after exclude: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# remove superfluous levels
try: self.prune()
except:
self.error( "%s: exception in pruning" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "pruning" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after pruning: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# remove group plots
try: self.group()
except:
self.error( "%s: exception in grouping" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "grouping" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after grouping: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
self.debug( "profile: started: renderer: %s" % (self.renderer))
try: result = self.render()
except:
self.error( "%s: exception in rendering" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "rendering" ) ))
finally:
self.debug( "profile: finished: renderer: %s" % (self.renderer))
#self.debug( "%s: heap at end\n%s" % (self, str(HP.heap()) ))
return result
def render( self ):
'''supply the :class:`Renderer.Renderer` with the data to render.
The data supplied will depend on the ``groupby`` option.
returns a ResultBlocks data structure.
'''
self.debug( "%s: rendering data started for %i items" % (self,
len(self.data)))
# get number of levels required by renderer
try:
renderer_nlevels = self.renderer.nlevels
except AttributeError:
# set to -1 to avoid any grouping
# important for user renderers that are functions
# and have no level attribute.
renderer_nlevels = -1
# initiate output structure
results = ResultBlocks( title = "")
# convert to data series
# The data is melted, i.e,
# BMW price 10000
# BMW speed 100
# Golf price 5000
# Golf speed 50
dataframe = DataTree.asDataFrame( self.data )
# dataframe.write_csv( "test.csv" )
if dataframe is None:
self.warn( "%s: no data after conversion" % self )
raise ValueError( "no data for renderer" )
# special patch: set column names to pruned levels
# if there are no column names
if len(dataframe.columns) == len(self.pruned):
if list(dataframe.columns) == list(range( len(dataframe.columns))):
dataframe.columns = [x[1] for x in self.pruned]
index = dataframe.index
def getIndexLevels( index ):
try:
# hierarchical index
nlevels = len(index.levels)
except AttributeError:
nlevels = 1
index = [ (x,) for x in index]
#raise ValueError('data frame without MultiIndex' )
return nlevels
nlevels = getIndexLevels( index )
self.debug( "%s: rendering data started. levels=%i, required levels>=%i, group_level=%s" %\
(self, nlevels,
renderer_nlevels,
str(self.group_level) ) )
if renderer_nlevels < 0 and self.group_level <= 0:
# no grouping for renderers that will accept
# a dataframe with any level of indices and no explicit
# grouping has been asked for.
results.append( self.renderer( dataframe, path = () ) )
else:
# user specified group level by default
group_level = self.group_level
# set group level to maximum allowed by renderer
if renderer_nlevels >= 0:
group_level = max(nlevels - renderer_nlevels, group_level)
# add additional level if necessary
if nlevels < group_level:
prefix = tuple(["level%i" % x for x in range( group_level - nlevels)])
dataframe.index = pandas.MultiIndex.from_tuples( [ prefix + x for x in dataframe.index ] )
# used to be: group_level + 1
# hierarchical index
# numpy.unique converts everything to a string
# which is not consistent with selecting later
paths = map( tuple, DataTree.unique( [ x[:group_level] for x in dataframe.index.unique() ] ))
pathlength = len(paths[0]) - 1
is_hierarchical = isinstance( dataframe.index, pandas.core.index.MultiIndex )
if is_hierarchical:
# Note: can only sort hierarchical indices
dataframe = dataframe.sortlevel()
if dataframe.index.lexsort_depth < pathlength:
raise ValueError('could not sort data frame: sort depth=%i < pathlength=%i, dataframe=%s' \
% (dataframe.index.lexsort_depth,
pathlength,
dataframe))
for path in paths:
if path:
if len(path) == nlevels:
# extract with loc in order to obtain dataframe
work = dataframe.loc[[path]]
else:
# select data frame as cross-section
work = dataframe.xs(path, axis=0 )
else:
# empty tuple - use full data set
work = dataframe
# remove columns and rows in work that are all Na
work = work.dropna( axis=1, how='all').dropna( axis=0, how='all')
if is_hierarchical and renderer_nlevels >= 0:
work_levels = getIndexLevels( work.index )
# reduce levels of indices required to that required
# for Renderer. This occurs if groupby=none.
if work_levels > renderer_nlevels:
sep = work_levels - (renderer_nlevels - 1)
tuples = [ ( DataTree.path2str( x[:sep] ), ) + x[sep:] \
for x in work.index ]
work.index = pandas.MultiIndex.from_tuples( tuples )
try:
results.append( self.renderer( work,
path = path ))
except:
self.error( "%s: exception in rendering" % self )
results.append( ResultBlocks( Utils.buildException( "rendering" ) ) )
if len(results) == 0:
self.warn("renderer returned no data.")
raise ValueError( "renderer returned no data." )
self.debug( "%s: rendering data finished with %i blocks" % (self.tracker, len(results)))
return results
