def __call__(self, data, path ):
lines = []
dirname = os.path.join( os.path.dirname(sys.modules["SphinxReport"].__file__), "images" )
descriptions = {}
title = "status"
# add header
lines.append( ".. csv-table:: %s" % "table" )
lines.append( ' :header: "Track", "Test", "", "Status", "Info" ' )
lines.append( '' )
for testname, w in data.iteritems():
for track, work in w.iteritems():
status = str(work['status']).strip()
descriptions[testname] = work['description']
info = str(work['info']).strip()
try:
image = ".. image:: %s" % os.path.join( dirname, self.map_code2image[status.upper()] )
except KeyError:
image = ""
lines.append( ' "%(track)s",":term:`%(testname)s`","%(image)s","%(status)s","%(info)s"' % locals() )
lines.append( "")
lines.append( ".. glossary::" )
lines.append( "" )
for test, description in descriptions.iteritems():
lines.append( '%s\n%s\n' % (Utils.indent(test,3), Utils.indent( description,6) ) )
return ResultBlocks( ResultBlock( "\n".join(lines), title = "") )
def getTrackers(fullpath):
"""retrieve an instantiated tracker and its associated code.
returns a tuple (code, tracker, module, flag).
The flag indicates whether that tracker is derived from the
tracker base class. If False, that tracker will not be listed
as an available tracker, though it might be specified at the
command line.
"""
name, cls = os.path.splitext(fullpath)
# remove leading '.'
cls = cls[1:]
module_name = os.path.basename(name)
module, pathname = Utils.getModule(name)
trackers = []
for name in dir(module):
obj = getattr(module, name)
try:
if Utils.isClass(obj):
trackers.append((name, obj, module_name, True))
else:
trackers.append((name, obj, module_name, False))
except ValueError:
pass
return trackers
def __call__(self, track, slice=None):
# note there are spaces behind the %(image)s directive to accomodate
# for path substitution
block = '''
.. figure:: %(image)s
:height: 300
'''
blocks = ResultBlocks()
tracks = sorted([x.asFile() for x in TRACKS])
for track in tracks:
files = glob.glob(
os.path.join(EXPORTDIR, "fastqc", "%s*_fastqc" % track))
for x, fn in enumerate(sorted(files)):
y = x + 1
image = os.path.abspath(
os.path.join(fn, "Images", "%s.png" % slice))
if not os.path.exists(image):
continue
blocks.append(
ResultBlock(text=block % locals(),
title=os.path.basename(fn)))
return odict(
(("rst", "\n".join(Utils.layoutBlocks(blocks,
layout="columns-2"))), ))
def writeCode( class_name, code, inliner ):
'''write code of class to file.
returns URI of written code.
'''
document = inliner.document.current_source
reference = class_name
# root of document tree
srcdir = setup.srcdir
# build directory
builddir = setup.confdir
# get the directory of the rst file
rstdir, rstfile = os.path.split( document )
basedir, fname, basename, ext, outdir, codename, notebookname = Utils.buildPaths( reference )
# path to root relative to rst
rst2srcdir = os.path.join( os.path.relpath( srcdir, start = rstdir ), outdir )
# output code
linked_codename = re.sub( "\\\\", "/", os.path.join( rst2srcdir, codename ))
if code and basedir != outdir:
outfile = open( os.path.join(outdir, codename ), "w" )
for line in code: outfile.write( line )
outfile.close()
return linked_codename
def transform(self, data, path ):
debug( "%s: called" % str(self))
if Utils.isArray( data ):
return Stats.Summary( data )._data
else:
return None
def transform(self, data, path ):
debug( "%s: called" % str(self))
t = odict()
for minor_key, values in data.items():
if not Utils.isArray(values): raise ValueError("expected a list for data frame creation, got %s", type(data))
if len(values) == 0: raise ValueError( "empty list for %s" % (minor_key))
v = values[0]
if Utils.isInt( v ):
t[minor_key] = rpy2.robjects.IntVector( values )
elif Utils.isFloat(v):
t[minor_key] = rpy2.robjects.FloatVector( values )
else:
t[minor_key] = rpy2.robjects.StrVector( values )
return rpy2.robjects.DataFrame(t)
def __call__(self, track, slice=None):
# note there are spaces behind the %(image)s directive to accomodate
# for path substitution
block = '''
.. figure:: %(image)s
:height: 300
'''
blocks = ResultBlocks()
tracks = sorted([x.asFile() for x in TRACKS])
for track in tracks:
files = glob.glob(
os.path.join(EXPORTDIR, "fastqc", "%s*_fastqc" % track))
for x, fn in enumerate(sorted(files)):
y = x + 1
image = os.path.abspath(
os.path.join(fn, "Images", "%s.png" % slice))
if not os.path.exists(image):
continue
blocks.append(ResultBlock(text=block % locals(),
title=os.path.basename(fn)))
return odict((("rst", "\n".join(Utils.layoutBlocks(blocks, layout="columns-2"))),))
def writeNotebook( outfile, options, kwargs,
renderer_options, transformer_options, display_options,
modulename, name):
'''write a snippet to paste with the ipython notebook.
'''
cmd_options = [
'do_print = False',
'tracker="%s"' % options.tracker,
'renderer="%s"' % options.renderer,
'trackerdir="%s"' % options.trackerdir,
'workdir="%s"' % os.getcwd() ]
for key, val in list(kwargs.items()) +\
list(renderer_options.items()) +\
list(transformer_options.items()):
if val == None:
cmd_options.append( "%s" % key )
else:
if Utils.isString( val ):
cmd_options.append( '%s="%s"' % (key,val) )
else:
cmd_options.append( '%s=%s' % (key,val) )
if options.transformers:
cmd_options.append( "transformer=['%s']" % "','".join( options.transformers))
# no module name in tracker
params = { "tracker" : "%s" % (name),
"options" : ",\n".join( cmd_options ) }
outfile.write( Utils.NOTEBOOK_TEMPLATE % params )
def __call__(self, dataframe, path ):
# convert to dataframe
# index has test names
# columns are description, info, status
columns = ('description', 'info', 'status', 'name')
if set(dataframe.columns) != set(columns):
raise ValueError( "invalid columns: expected '%s', got '%s' " %\
(columns, dataframe.columns))
lines = []
dirname = os.path.join( os.path.dirname(sys.modules["SphinxReport"].__file__), "images" )
descriptions = {}
title = "status"
# add header
lines.append( ".. csv-table:: %s" % "table" )
lines.append( " :class: sortable" )
lines.append( ' :header: "Track", "Test", "", "Status", "Info" ' )
lines.append( '' )
for index, values in dataframe.iterrows():
testname=values['name']
description=values['description']
info = values['info']
status = values['status']
track = index
descriptions[testname] = description
try:
image = ".. image:: %s" % os.path.join( dirname, self.map_code2image[status.upper()] )
except KeyError:
image = ""
lines.append( ' "%(track)s",":term:`%(testname)s`","%(image)s","%(status)s","%(info)s"' % locals() )
lines.append( "")
lines.append( ".. glossary::" )
lines.append( "" )
for test, description in descriptions.items():
lines.append( '%s\n%s\n' % (Utils.indent(test,3), Utils.indent( description,6) ) )
return ResultBlocks( ResultBlock( "\n".join(lines), title = "") )
def path2str( path ):
'''convert path to printable string.'''
if path is None: return ""
if Utils.isString( path ): return path
try:
return "/".join(map(str,path))
except:
return str(path)
def copy( src, dst ):
fn = os.path.join( dest, dst, src)
if os.path.exists( fn ):
raise OSError( "file %s already exists - not overwriting." % fn )
outfile = open( fn, "w" )
x = Utils.get_data( "SphinxReport", "templates/%s" % src)
outfile.write( x )
outfile.close()
def buildPaths( reference ):
'''return paths.'''
basedir, fname = os.path.split(reference)
basename, ext = os.path.splitext(fname)
outdir = os.path.join('_static', 'report_directive', basedir)
codename = Utils.quote_filename(reference) + ".code"
return basedir, fname, basename, ext, outdir, codename
def render(self):
"""supply the :class:`Renderer.Renderer` with the data to render.
The data supplied will depend on the ``groupby`` option.
return resultblocks
"""
self.debug("%s: rendering data started for %i items" % (self, len(self.data)))
results = ResultBlocks(title="main")
# get number of levels required by renderer
try:
renderer_nlevels = self.renderer.nlevels
except AttributeError:
renderer_nlevels = 0
data_paths = DataTree.getPaths(self.data)
nlevels = len(data_paths)
group_level = self.group_level
self.debug(
"%s: rendering data started. levels=%i, required levels>=%i, group_level=%i, data_paths=%s"
% (self, nlevels, renderer_nlevels, group_level, str(data_paths)[:100])
)
if nlevels < renderer_nlevels:
# add some dummy levels if levels is not enough
d = self.data
for x in range(renderer_nlevels - nlevels):
d = odict((("all", d),))
results.append(self.renderer(d, path=("all",)))
elif group_level < 0 or renderer_nlevels < 0:
# no grouping
results.append(self.renderer(self.data, path=()))
else:
# group at level group_level
paths = list(itertools.product(*data_paths[: group_level + 1]))
for path in paths:
work = DataTree.getLeaf(self.data, path)
if not work:
continue
try:
results.append(self.renderer(work, path=path))
except:
results.append(ResultBlocks(Utils.buildException("rendering")))
if len(results) == 0:
self.warn("tracker returned no data.")
raise ValueError("tracker returned no data.")
self.debug("%s: rendering data finished with %i blocks" % (self.tracker, len(results)))
return results
def tracker2key(tracker):
"""derive cache filename from a tracker."""
modulename = os.path.split(tracker.__class__.__module__)[1]
if hasattr(tracker, "func_name"):
name = tracker.func_name
else:
name = tracker.__class__.__name__
return Utils.quote_filename(".".join((modulename, name)))
def tracker2key( tracker ):
'''derive cache filename from a tracker.'''
modulename = tracker.__module__
try:
# works for functions (def)
name = tracker.__name__
except AttributeError:
# works for functors (class)
name = tracker.__class__.__name__
return Utils.quote_filename( ".".join((modulename,name)))
def __call__(self, track, slice = None ):
blocks = ResultBlocks()
block = '''
.. figure:: %(image)s
:height: 300
'''
for image in glob.glob( os.path.join( IMAGEDIR, "*.png" )):
blocks.append( ResultBlock( text = block % locals(),
title = "image" ) )
return odict( (("rst", "\n".join( Utils.layoutBlocks( blocks, layout = "columns-2"))),))
def transform(self, data, path):
debug( "%s: called for path %s" % (str(self), str(path)))
if not Utils.isArray( data ): return None
bins, values = self.toHistogram(data)
if bins != None:
for converter in self.mConverters: values = converter(values)
debug( "%s: completed for path %s" % (str(self), str(path)))
header = "bins"
#if len(path) > 1: header = path[-1]
#else: header = "bins"
return odict( ((header, bins), ("frequency", values)))
def __call__(self, *args, **kwargs):
try:
self.parseArguments(*args, **kwargs)
except:
return ResultBlocks(ResultBlocks(Utils.buildException("parsing")))
self.debug("profile: started: tracker: %s" % (self.tracker))
try:
self.collect()
except:
return ResultBlocks(ResultBlocks(Utils.buildException("collection")))
self.debug("profile: finished: tracker: %s" % (self.tracker))
data_paths = DataTree.getPaths(self.data)
self.debug("%s: after collection: %i data_paths: %s" % (self, len(data_paths), str(data_paths)))
# transform data
try:
self.transform()
except:
return ResultBlocks(ResultBlocks(Utils.buildException("transformation")))
data_paths = DataTree.getPaths(self.data)
self.debug("%s: after transformation: %i data_paths: %s" % (self, len(data_paths), str(data_paths)))
# remove superfluous levels
try:
self.prune()
except:
return ResultBlocks(ResultBlocks(Utils.buildException("pruning")))
data_paths = DataTree.getPaths(self.data)
self.debug("%s: after pruning: %i data_paths: %s" % (self, len(data_paths), str(data_paths)))
# remove group plots
try:
self.group()
except:
return ResultBlocks(ResultBlocks(Utils.buildException("grouping")))
data_paths = DataTree.getPaths(self.data)
self.debug("%s: after grouping: %i data_paths: %s" % (self, len(data_paths), str(data_paths)))
self.debug("profile: started: renderer: %s" % (self.renderer))
try:
result = self.render()
except:
return ResultBlocks(ResultBlocks(Utils.buildException("rendering")))
self.debug("profile: finished: renderer: %s" % (self.renderer))
return result
def param_role( role, rawtext, text, lineno, inliner,
options={}, content=[]):
'''inserts a member variable of a tracker class in the text.'''
parts = text.split(".")
if len(parts) < 2:
msg = inliner.reporter.error(
':param: should be in class.value format '
': "%s" is invalid.' % text, line=lineno)
prb = inliner.problematic(rawtext, rawtext, msg)
return [prb], [msg]
class_name, parameter_name = ".".join(parts[:-1]), parts[-1]
try:
code, tracker = Utils.makeTracker( class_name )
except AttributeError:
tracker = None
if not tracker:
msg = inliner.reporter.error(
':param: can not find class '
'"%s".' % class_name, line=lineno)
prb = inliner.problematic(rawtext, rawtext, msg)
return [prb], [msg]
try:
value = getattr( tracker, parameter_name )
except AttributeError:
msg = inliner.reporter.error(
':param: can not find variable %s in '
': "%s" is invalid -tracker=%s.' % (parameter_name, class_name, str(tracker)), line=lineno)
prb = inliner.problematic(rawtext, rawtext, msg)
return [prb], [msg]
linked_codename = writeCode( class_name, code, inliner )
node = nodes.reference(rawtext,
utils.unescape(str(value)),
refuri=linked_codename,
**options)
return [node], []
def value_role( role, rawtext, text, lineno, inliner,
options={}, content=[]):
'''insert a single value from a tracker into text.'''
class_name = text
try:
code, tracker = Utils.makeTracker( class_name )
except (AttributeError, ImportError):
tracker = None
if not tracker:
msg = inliner.reporter.error(
':value: can not find class '
'"%s".' % class_name, line=lineno)
prb = inliner.problematic(rawtext, rawtext, msg)
return [prb], [msg]
# Python 2/3
try:
value = str( tracker() )
except TypeError as msg:
print("python 3 problem: %s: tracker=%s" % (msg, str(tracker()) ))
linked_codename = writeCode( class_name, code, inliner )
# Base URL mainly used by inliner.rfc_reference, so this is correct:
# ref = inliner.document.settings.rfc_base_url + inliner.rfc_url % value
# in example, but deprecated
# set_classes(options)
#node = nodes.literal(rawtext,
# utils.unescape(str(value)),
# **options)
node = nodes.reference(rawtext,
utils.unescape(str(value)),
refuri=linked_codename,
**options)
return [node], []
def param_role(role, rawtext, text, lineno, inliner, options={}, content=[]):
parts = text.split(".")
if len(parts) < 2:
msg = inliner.reporter.error(
":param: should be in class.value format " ': "%s" is invalid.' % text, line=lineno
)
prb = inliner.problematic(rawtext, rawtext, msg)
return [prb], [msg]
class_name, parameter_name = ".".join(parts[:-1]), parts[-1]
try:
code, tracker = Utils.makeTracker(class_name)
except AttributeError:
tracker = None
if not tracker:
msg = inliner.reporter.error(":param: can not find class " '"%s".' % class_name, line=lineno)
prb = inliner.problematic(rawtext, rawtext, msg)
return [prb], [msg]
try:
value = getattr(tracker, parameter_name)
except AttributeError:
msg = inliner.reporter.error(
":param: can not find variable %s in "
': "%s" is invalid -tracker=%s.' % (parameter_name, class_name, str(tracker)),
line=lineno,
)
prb = inliner.problematic(rawtext, rawtext, msg)
return [prb], [msg]
# Base URL mainly used by inliner.rfc_reference, so this is correct:
# ref = inliner.document.settings.rfc_base_url + inliner.rfc_url % value
# in example, but deprecated
# set_classes(options)
node = nodes.literal(rawtext, utils.unescape(str(value)), **options)
return [node], []
def collect( self,
blocks,
template_name,
outdir,
rstdir,
builddir,
srcdir,
content,
display_options,
tracker_id,
links = {}):
'''collect html output from result blocks.
HTML output is written to a file and a link will be inserted at
the place holder.
'''
map_figure2text = {}
extension = "html"
for xblocks in blocks:
for block in xblocks:
if not hasattr( block, "html" ): continue
# remove special characters from filename. I think the docutils
# system removes these from links which later causes problems
# as the link does not point to the correct location of the file.
outname = Utils.quote_filename( "%s_%s" % (template_name, block.title) )
outputpath = os.path.join(outdir, '%s.%s' % (outname, extension))
# save to file
outf = open( outputpath, "w")
outf.write( block.html )
outf.close()
# use absolute path
link = os.path.abspath( outputpath )
rst_output = "%(link)s" % locals()
map_figure2text[ "#$html %s$#" % block.title] = rst_output
return map_figure2text
def render(self, work, path ):
self.startPlot()
plts, legend = [], []
all_data = []
for line, data in work.iteritems():
assert len(data) == 1, "multicolumn data not supported yet: %s" % str(data)
for label, values in data.iteritems():
assert Utils.isArray( values ), "work is of type '%s'" % values
d = [ x for x in values if x != None ]
if len(d) > 0:
all_data.append( ro.FloatVector( d ) )
legend.append( "/".join((line,label)))
R.boxplot( all_data )
return self.endPlot( work, path )
def value_role(role, rawtext, text, lineno, inliner, options={}, content=[]):
class_name = text
try:
code, tracker = Utils.makeTracker(class_name)
except (AttributeError, ImportError):
tracker = None
if not tracker:
msg = inliner.reporter.error(":value: can not find class " '"%s".' % class_name, line=lineno)
prb = inliner.problematic(rawtext, rawtext, msg)
return [prb], [msg]
value = str(tracker())
# Base URL mainly used by inliner.rfc_reference, so this is correct:
# ref = inliner.document.settings.rfc_base_url + inliner.rfc_url % value
# in example, but deprecated
# set_classes(options)
node = nodes.literal(rawtext, utils.unescape(str(value)), **options)
return [node], []
(reference, basedir, rst2srcdir, rst2builddir, outdir, codename))
# try to create. If several processes try to create it,
# testing with `if` will not work.
try:
os.makedirs(outdir)
except OSError, msg:
pass
if not os.path.exists(outdir):
raise OSError( "could not create directory %s: %s" % (outdir, msg ))
########################################################
# collect options
# replace placedholders
options = Utils.updateOptions( options )
logging.debug( "report_directive.run: updated options=%s" % (str(options),) )
transformer_names = []
renderer_name = None
# get layout option
layout = options.get( "layout", "column" )
option_map = getOptionMap()
renderer_options = selectAndDeleteOptions( options, option_map["render"])
transformer_options = selectAndDeleteOptions( options, option_map["transform"])
dispatcher_options = selectAndDeleteOptions( options, option_map["dispatch"] )
logging.debug( "report_directive.run: renderer options: %s" % str(renderer_options) )
logging.debug( "report_directive.run: transformer options: %s" % str(transformer_options) )
def collect(
self, blocks, template_name, outdir, rstdir, builddir, srcdir, content, display_options, tracker_id, links={}
):
"""collect one or more matplotlib figures and
1. save as png, hires-png and pdf
2. save thumbnail
3. insert rendering code at placeholders in output
returns a map of place holder to placeholder text.
"""
fig_managers = _pylab_helpers.Gcf.get_all_fig_managers()
map_figure2text = {}
# determine the image formats to create
default_format, additional_formats = Utils.getImageFormats(display_options)
all_formats = [default_format] + additional_formats
# create all the images
for figman in fig_managers:
# create all images
figid = figman.num
outname = "%s_%02d" % (template_name, figid)
for id, format, dpi in all_formats:
outpath = os.path.join(outdir, "%s.%s" % (outname, format))
try:
figman.canvas.figure.savefig(outpath, dpi=dpi)
except:
s = Utils.collectExceptionAsString("Exception running plot %s" % outpath)
warnings.warn(s)
return []
# if format=='png':
# thumbdir = os.path.join(outdir, 'thumbnails')
# try:
# os.makedirs(thumbdir)
# except OSError:
# pass
# thumbfile = str('%s.png' % os.path.join(thumbdir, outname) )
# captionfile = str('%s.txt' % os.path.join(thumbdir, outname) )
# if not os.path.exists(thumbfile):
# # thumbnail only available in matplotlib >= 0.98.4
# try:
# figthumb = image.thumbnail(str(outpath), str(thumbfile), scale=0.3)
# except AttributeError:
# pass
# outfile = open(captionfile,"w")
# outfile.write( "\n".join( content ) + "\n" )
# outfile.close()
# create the text element
rst_output = Utils.buildRstWithImage(
outname,
outdir,
rstdir,
builddir,
srcdir,
additional_formats,
tracker_id,
links,
display_options,
default_format,
)
map_figure2text["#$mpl %i$#" % figid] = rst_output
return map_figure2text
def collect( self,
blocks,
template_name,
outdir,
rstdir,
rst2rootdir,
rst2builddir,
rst2srcdir,
content,
display_options,
linked_codename,
tracker_id):
'''collect one or more R figures from all active devices
and
1. save as png, hires-png and pdf
2. save thumbnail
3. insert rendering code at placeholders in output
returns a map of place holder to placeholder text.
'''
map_figure2text = {}
# determine the image formats to create
default_format, additional_formats = Utils.getImageFormats()
all_formats = [default_format,] + additional_formats
urls = Utils.asList( Utils.PARAMS["report_urls"] )
devices = R["dev.list"]()
try:
maxid = max( R["dev.list"]() )
except TypeError:
return map_figure2text
for figid in range( 2, maxid+1 ):
for id, format, dpi in all_formats:
R["dev.set"]( figid )
outname = "%s_%02d" % (template_name, figid)
outpath = os.path.join(outdir, '%s.%s' % (outname, format))
if format.endswith( "png" ):
R["dev.copy"]( device = R.png,
filename = outpath,
res = dpi )
R["dev.off"]()
elif format.endswith( "svg" ):
R["dev.copy"]( device = R.svg,
filename = outpath )
R["dev.off"]()
elif format.endswith( "eps" ):
R["dev.copy"]( device = R.postscript,
file = outpath,
onefile = True )
R["dev.off"]()
else:
raise ValueError( "format '%s' not supported" % format )
if not os.path.exists( outpath ):
continue
# raise ValueError( "rendering problem: image file was not be created: %s" % outpath )
if format=='png':
thumbdir = os.path.join(outdir, 'thumbnails')
try:
os.makedirs(thumbdir)
except OSError:
pass
thumbfile = str('%s.png' % os.path.join(thumbdir, outname) )
captionfile = str('%s.txt' % os.path.join(thumbdir, outname) )
if not os.path.exists(thumbfile):
# thumbnail only available in matplotlib >= 0.98.4
try:
figthumb = image.thumbnail(str(outpath), str(thumbfile), scale=0.3)
except AttributeError:
pass
outfile = open(captionfile,"w")
outfile.write( "\n".join( content ) + "\n" )
outfile.close()
R["dev.off"](figid)
# create the text element
rst_output = ""
# for image diretive - image path is relative from rst file to external build dir
imagepath = re.sub( "\\\\", "/", os.path.join( rst2builddir, outname ) )
# for links - path is from rst file to internal root dir
relative_imagepath = re.sub( "\\\\", "/", os.path.join( rst2rootdir, outname ) )
linked_text = relative_imagepath + ".txt"
if Config.HTML_IMAGE_FORMAT:
id, format, dpi = Config.HTML_IMAGE_FORMAT
template = '''
.. htmlonly::
.. image:: %(linked_image)s
%(display_options)s
[%(code_url)s %(rst_url)s %(data_url)s %(extra_images)s]
'''
linked_image = imagepath + ".%s" % format
extra_images=[]
for id, format, dpi in additional_formats:
extra_images.append( "`%(id)s <%(relative_imagepath)s.%(format)s>`__" % locals())
if extra_images: extra_images = " " + " ".join( extra_images)
else: extra_images = ""
# construct additional urls
code_url, data_url, rst_url = "", "", ""
if "code" in urls:
code_url = "`code <%(linked_codename)s>`__" % locals()
if "data" in urls:
data_url = "`data </data/%(tracker_id)s>`__" % locals()
if "rst" in urls:
rst_url = "`rst <%(linked_text)s>`__" % locals()
rst_output += template % locals()
# treat latex separately
if Config.LATEX_IMAGE_FORMAT:
id, format, dpi = Config.LATEX_IMAGE_FORMAT
template = '''
.. latexonly::
.. image:: %(linked_image)s
%(display_options)s
'''
linked_image = imagepath + ".%s" % format
rst_output += template % locals()
map_figure2text[ "#$rpl %i$#" % figid] = rst_output
return map_figure2text
def main():
parser = optparse.OptionParser(version="%prog version: $Id$", usage=USAGE)
parser.add_option("-t", "--tracker", dest="tracker", type="string", help="tracker to use [default=%default]")
parser.add_option("-p", "--page", dest="page", type="string", help="render an rst page [default=%default]")
parser.add_option("-a", "--tracks", dest="tracks", type="string", help="tracks to use [default=%default]")
parser.add_option(
"-m",
"--transformer",
dest="transformers",
type="string",
action="append",
help="add transformation [default=%default]",
)
parser.add_option("-s", "--slices", dest="slices", type="string", help="slices to use [default=%default]")
parser.add_option("-r", "--renderer", dest="renderer", type="string", help="renderer to use [default=%default]")
parser.add_option("-w", "--path", dest="dir_trackers", type="string", help="path to trackers [default=%default]")
parser.add_option(
"-f",
"--force",
dest="force",
action="store_true",
help="force recomputation of data by deleting cached results [default=%default]",
)
parser.add_option(
"-o",
"--option",
dest="options",
type="string",
action="append",
help="renderer options - supply as key=value pairs (without spaces). [default=%default]",
)
parser.add_option(
"--no-print",
dest="do_print",
action="store_false",
help="do not print an rst text element to create the displayed plots [default=%default].",
)
parser.add_option("--no-show", dest="do_show", action="store_false", help="do not show a plot [default=%default].")
parser.set_defaults(
tracker=None,
transformers=[],
tracks=None,
slices=None,
options=[],
renderer=None,
do_show=True,
do_print=True,
force=False,
dir_trackers=TRACKERDIR,
label="GenericLabel",
caption="add caption here",
)
(options, args) = parser.parse_args()
if len(args) == 2:
options.tracker, options.renderer = args
# configure options
options.dir_trackers = os.path.abspath(os.path.expanduser(options.dir_trackers))
if not os.path.exists(options.dir_trackers):
raise IOError("directory %s does not exist" % options.dir_trackers)
sys.path.append(options.dir_trackers)
# test plugins
kwargs = {}
for x in options.options:
if "=" in x:
data = x.split("=")
key, val = [y.strip() for y in (data[0], "=".join(data[1:]))]
else:
key, val = x.strip(), None
kwargs[key] = val
if options.tracks:
kwargs["tracks"] = options.tracks
if options.slices:
kwargs["slices"] = options.slices
if options.renderer == None:
options.renderer = "table"
kwargs = Utils.updateOptions(kwargs)
renderer = Utils.getRenderer(options.renderer, **kwargs)
transformers = Utils.getTransformers(options.transformers, **kwargs)
exclude = set(
(
"Tracker",
"TrackerSQL",
"returnLabeledData",
"returnMultipleColumnData",
"returnMultipleColumns",
"returnSingleColumn",
"returnSingleColumnData",
"SQLError",
"MultipleColumns",
"MultipleColumnData",
"LabeledData",
"DataSimple",
"Data",
)
)
if options.tracker:
trackers = []
for filename in glob.glob(os.path.join(options.dir_trackers, "*.py")):
modulename = os.path.basename(filename)
trackers.extend([x for x in getTrackers(modulename) if x[0] not in exclude])
for name, tracker, modulename, is_derived in trackers:
if name == options.tracker:
break
else:
available_trackers = set([x[0] for x in trackers if x[3]])
print "unknown tracker '%s': possible trackers are\n %s" % (
options.tracker,
"\n ".join(sorted(available_trackers)),
)
print "(the list above does not contain functions)."
sys.exit(1)
## remove everything related to that tracker for a clean slate
if options.force:
removed = SphinxReport.clean.removeTracker(name)
print "removed all data for tracker %s: %i files" % (name, len(removed))
# instantiate functors
if is_derived:
t = tracker()
# but not functions
else:
t = tracker
dispatcher = Dispatcher(t, renderer, transformers)
## needs to be resolved between renderer and dispatcher options
result = dispatcher(**kwargs)
if options.do_print:
options_rst = []
for key, val in kwargs.items():
if val == None:
options_rst.append(":%s:" % key)
else:
options_rst.append(":%s: %s" % (key, val))
print "..Template start"
print
params = {
"tracker": "%s.%s" % (modulename, name),
"renderer": options.renderer,
"label": options.label,
"options": ("\n ").join(options_rst),
"caption": options.caption,
}
if options.transformers:
params["options"] = ":transform: %s\n %s" % (",".join(options.transformers), params["options"])
print RST_TEMPLATE % params
print
print "..Template ends"
if result:
for r in result:
print "title:", r.title
for s in r:
print str(s)
if options.do_show:
if options.renderer.startswith("r-"):
print "press Ctrl-c to stop"
while 1:
pass
elif _pylab_helpers.Gcf.get_all_fig_managers() > 0:
plt.show()
elif options.page:
import build
SphinxReport.report_directive.DEBUG = True
SphinxReport.report_directive.FORCE = True
blocks = build.rst_reader(open(options.page, "r"))
for block in blocks:
build.run(((options.page, block),))
def run(arguments, options, lineno, content, state_machine=None, document=None, srcdir=None, builddir=None):
"""process :report: directive.
*srdir* - top level directory of rst documents
*builddir* - build directory
"""
tag = "%s:%i" % (str(document), lineno)
logging.debug("report_directive.run: profile: started: rst: %s" % tag)
# sort out the paths
# reference is used for time-stamping
tracker_name = directives.uri(arguments[0])
basedir, fname, basename, ext, outdir, codename, notebookname = Utils.buildPaths(tracker_name)
# get the directory of the rst file
rstdir, rstfile = os.path.split(document) # state_machine.document.attributes['source'])
# root of document tree
if srcdir is None:
srcdir = setup.srcdir
# build directory
if builddir is None:
builddir = setup.confdir
# remove symbolic links
srcdir, builddir, rstdir = [os.path.realpath(x) for x in (srcdir, builddir, rstdir)]
# there are three directories:
# builddir = directory where document is built in (usually _build/html or similar)
# rstdir = directory where rst sources are located
# srcdir = directory from which the build process is started
# path to root relative to rst
rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir)
# path to root relative to rst
rst2builddir = os.path.join(os.path.relpath(builddir, start=rstdir), outdir)
# path relative to source (for images)
root2builddir = os.path.join(os.path.relpath(builddir, start=srcdir), outdir)
logging.debug(
"report_directive.run: arguments=%s, options=%s, lineno=%s, content=%s, document=%s"
% (str(arguments), str(options), str(lineno), str(content), str(document))
)
logging.debug(
"report_directive.run: plotdir=%s, basename=%s, ext=%s, fname=%s, rstdir=%s, srcdir=%s, builddir=%s"
% (tracker_name, basename, ext, fname, rstdir, srcdir, builddir)
)
logging.debug(
"report_directive.run: tracker_name=%s, basedir=%s, rst2src=%s, root2build=%s, outdir=%s, codename=%s"
% (tracker_name, basedir, rst2srcdir, rst2builddir, outdir, codename)
)
# try to create. If several processes try to create it,
# testing with `if` will not work.
try:
os.makedirs(outdir)
except OSError as msg:
pass
if not os.path.exists(outdir):
raise OSError("could not create directory %s: %s" % (outdir, msg))
########################################################
# collect options
# replace placedholders
try:
options = Utils.updateOptions(options)
except ValueError as msg:
logging.warn("failure while updating options: %s" % msg)
logging.debug("report_directive.run: options=%s" % (str(options),))
transformer_names = []
renderer_name = None
# get layout option
layout = options.get("layout", "column")
option_map = getOptionMap()
renderer_options = Utils.selectAndDeleteOptions(options, option_map["render"])
transformer_options = Utils.selectAndDeleteOptions(options, option_map["transform"])
dispatcher_options = Utils.selectAndDeleteOptions(options, option_map["dispatch"])
tracker_options = Utils.selectAndDeleteOptions(options, option_map["tracker"])
display_options = Utils.selectAndDeleteOptions(options, option_map["display"])
logging.debug("report_directive.run: renderer options: %s" % str(renderer_options))
logging.debug("report_directive.run: transformer options: %s" % str(transformer_options))
logging.debug("report_directive.run: dispatcher options: %s" % str(dispatcher_options))
logging.debug("report_directive.run: tracker options: %s" % str(tracker_options))
logging.debug("report_directive.run: display options: %s" % str(display_options))
if "transform" in display_options:
transformer_names = display_options["transform"].split(",")
del display_options["transform"]
if "render" in display_options:
renderer_name = display_options["render"]
del display_options["render"]
########################################################
# check for missing files
if renderer_name != None:
options_key = (
str(renderer_options)
+ str(transformer_options)
+ str(dispatcher_options)
+ str(tracker_options)
+ str(transformer_names)
)
options_hash = hashlib.md5(options_key.encode()).hexdigest()
template_name = Utils.quote_filename(Config.SEPARATOR.join((tracker_name, renderer_name, options_hash)))
filename_text = os.path.join(outdir, "%s.txt" % (template_name))
logging.debug("report_directive.run: options_hash=%s" % options_hash)
###########################################################
# check for existing files
# update strategy does not use file stamps, but checks
# for presence/absence of text element and if all figures
# mentioned in the text element are present
###########################################################
queries = [re.compile("%s(%s\S+.%s)" % (root2builddir, outdir, suffix)) for suffix in ("png", "pdf")]
logging.debug("report_directive.run: checking for changed files.")
# check if text element exists
if os.path.exists(filename_text):
lines = [x[:-1] for x in open(filename_text, "r").readlines()]
filenames = []
# check if all figures are present
for line in lines:
for query in queries:
x = query.search(line)
if x:
filenames.extend(list(x.groups()))
logging.debug("report_directive.run: %s: checking for %s" % (tag, str(filenames)))
for filename in filenames:
if not os.path.exists(filename):
logging.info("report_directive.run: %s: redo: %s missing" % (tag, filename))
break
else:
logging.info("report_directive.run: %s: noredo: all files are present" % tag)
## all is present - save text and return
if lines and state_machine:
state_machine.insert_input(lines, state_machine.input_lines.source(0))
return []
else:
logging.debug("report_directive.run: %s: no check performed: %s missing" % (tag, str(filename_text)))
else:
template_name = ""
filename_text = None
##########################################################
# Initialize collectors
collectors = []
for collector in list(getPlugins("collect").values()):
collectors.append(collector())
##########################################################
## instantiate tracker, dispatcher, renderer and transformers
## and collect output
###########################################################
try:
########################################################
# find the tracker
logging.debug("report_directive.run: collecting tracker %s with options %s " % (tracker_name, tracker_options))
code, tracker = Utils.makeTracker(tracker_name, (), tracker_options)
if not tracker:
logging.error("report_directive.run: no tracker - no output from %s " % str(document))
raise ValueError("tracker `%s` not found" % tracker_name)
logging.debug("report_directive.run: collected tracker %s" % tracker_name)
tracker_id = Cache.tracker2key(tracker)
########################################################
# determine the transformer
logging.debug("report_directive.run: creating transformers")
transformers = Utils.getTransformers(transformer_names, transformer_options)
########################################################
# determine the renderer
logging.debug("report_directive.run: creating renderer.")
if renderer_name == None:
logging.error("report_directive.run: no renderer - no output from %s" % str(document))
raise ValueError("the report directive requires a renderer")
renderer = Utils.getRenderer(renderer_name, renderer_options)
########################################################
# create and call dispatcher
logging.debug("report_directive.run: creating dispatcher")
dispatcher = Dispatcher.Dispatcher(tracker, renderer, transformers)
# add the tracker options
dispatcher_options.update(tracker_options)
blocks = dispatcher(**dispatcher_options)
if blocks == None:
blocks = ResultBlocks(
ResultBlocks(Utils.buildWarning("NoData", "tracker %s returned no Data" % str(tracker)))
)
code = None
tracker_id = None
except:
logging.warn("report_directive.run: exception caught at %s:%i - see document" % (str(document), lineno))
blocks = ResultBlocks(ResultBlocks(Utils.buildException("invocation")))
code = None
tracker_id = None
logging.debug("report_directive.run: profile: started: collecting: %s" % tag)
########################################################
## write code output
linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename))
if code and basedir != outdir:
with open(os.path.join(outdir, codename), "w") as outfile:
for line in code:
outfile.write(line)
########################################################
## write notebook snippet
linked_notebookname = re.sub("\\\\", "/", os.path.join(rst2srcdir, notebookname))
if basedir != outdir:
with open(os.path.join(outdir, notebookname), "w") as outfile:
Utils.writeNoteBookEntry(
outfile,
renderer=renderer_name,
tracker=tracker_name,
transformers=transformer_names,
options=renderer_options.items() + tracker_options.items() + transformer_options.items(),
)
###########################################################
# collect images
###########################################################
map_figure2text = {}
try:
for collector in collectors:
map_figure2text.update(
collector.collect(
blocks,
template_name,
outdir,
rstdir,
builddir,
srcdir,
content,
display_options,
tracker_id,
links={"code_url": linked_codename, "notebook_url": linked_notebookname},
)
)
except:
logging.warn(
"report_directive.run: exception caught while collecting with %s at %s:%i - see document"
% (collector, str(document), lineno)
)
blocks = ResultBlocks(ResultBlocks(Utils.buildException("collection")))
code = None
tracker_id = None
###########################################################
# replace place holders or add text
###########################################################
## add default for text-only output
map_figure2text["default-prefix"] = TEMPLATE_TEXT % locals()
map_figure2text["default-suffix"] = ""
blocks.updatePlaceholders(map_figure2text)
###########################################################
## render the output taking into account the layout
lines = Utils.layoutBlocks(blocks, layout)
###########################################################
# add caption
lines.extend(["::", ""])
if content:
lines.extend([" %s" % row.strip() for row in content])
lines.append("")
lines.append("")
# output rst text for this renderer
if filename_text:
outfile = open(filename_text, "w")
outfile.write("\n".join(lines))
outfile.close()
if SPHINXREPORT_DEBUG:
for x, l in enumerate(lines):
print("%5i %s" % (x, l))
if len(lines) and state_machine:
state_machine.insert_input(lines, state_machine.input_lines.source(0))
logging.debug("report_directive.run: profile: finished: collecting: %s" % tag)
logging.debug("report_directive.run: profile: finished: rst: %s:%i" % (str(document), lineno))
return []
def render( self ):
'''supply the :class:`Renderer.Renderer` with the data to render.
The data supplied will depend on the ``groupby`` option.
returns a ResultBlocks data structure.
'''
self.debug( "%s: rendering data started for %i items" % (self,
len(self.data)))
# get number of levels required by renderer
try:
renderer_nlevels = self.renderer.nlevels
except AttributeError:
# set to -1 to avoid any grouping
# important for user renderers that are functions
# and have no level attribute.
renderer_nlevels = -1
# initiate output structure
results = ResultBlocks( title = "")
# convert to data series
# The data is melted, i.e,
# BMW price 10000
# BMW speed 100
# Golf price 5000
# Golf speed 50
dataframe = DataTree.asDataFrame( self.data )
# dataframe.write_csv( "test.csv" )
if dataframe is None:
self.warn( "%s: no data after conversion" % self )
raise ValueError( "no data for renderer" )
# special patch: set column names to pruned levels
# if there are no column names
if len(dataframe.columns) == len(self.pruned):
if list(dataframe.columns) == list(range( len(dataframe.columns))):
dataframe.columns = [x[1] for x in self.pruned]
index = dataframe.index
def getIndexLevels( index ):
try:
# hierarchical index
nlevels = len(index.levels)
except AttributeError:
nlevels = 1
index = [ (x,) for x in index]
#raise ValueError('data frame without MultiIndex' )
return nlevels
nlevels = getIndexLevels( index )
self.debug( "%s: rendering data started. levels=%i, required levels>=%i, group_level=%s" %\
(self, nlevels,
renderer_nlevels,
str(self.group_level) ) )
if renderer_nlevels < 0 and self.group_level <= 0:
# no grouping for renderers that will accept
# a dataframe with any level of indices and no explicit
# grouping has been asked for.
results.append( self.renderer( dataframe, path = () ) )
else:
# user specified group level by default
group_level = self.group_level
# set group level to maximum allowed by renderer
if renderer_nlevels >= 0:
group_level = max(nlevels - renderer_nlevels, group_level)
# add additional level if necessary
if nlevels < group_level:
prefix = tuple(["level%i" % x for x in range( group_level - nlevels)])
dataframe.index = pandas.MultiIndex.from_tuples( [ prefix + x for x in dataframe.index ] )
# used to be: group_level + 1
# hierarchical index
# numpy.unique converts everything to a string
# which is not consistent with selecting later
paths = map( tuple, DataTree.unique( [ x[:group_level] for x in dataframe.index.unique() ] ))
pathlength = len(paths[0]) - 1
is_hierarchical = isinstance( dataframe.index, pandas.core.index.MultiIndex )
if is_hierarchical:
# Note: can only sort hierarchical indices
dataframe = dataframe.sortlevel()
if dataframe.index.lexsort_depth < pathlength:
raise ValueError('could not sort data frame: sort depth=%i < pathlength=%i, dataframe=%s' \
% (dataframe.index.lexsort_depth,
pathlength,
dataframe))
for path in paths:
if path:
if len(path) == nlevels:
# extract with loc in order to obtain dataframe
work = dataframe.loc[[path]]
else:
# select data frame as cross-section
work = dataframe.xs(path, axis=0 )
else:
# empty tuple - use full data set
work = dataframe
# remove columns and rows in work that are all Na
work = work.dropna( axis=1, how='all').dropna( axis=0, how='all')
if is_hierarchical and renderer_nlevels >= 0:
work_levels = getIndexLevels( work.index )
# reduce levels of indices required to that required
# for Renderer. This occurs if groupby=none.
if work_levels > renderer_nlevels:
sep = work_levels - (renderer_nlevels - 1)
tuples = [ ( DataTree.path2str( x[:sep] ), ) + x[sep:] \
for x in work.index ]
work.index = pandas.MultiIndex.from_tuples( tuples )
try:
results.append( self.renderer( work,
path = path ))
except:
self.error( "%s: exception in rendering" % self )
results.append( ResultBlocks( Utils.buildException( "rendering" ) ) )
if len(results) == 0:
self.warn("renderer returned no data.")
raise ValueError( "renderer returned no data." )
self.debug( "%s: rendering data finished with %i blocks" % (self.tracker, len(results)))
return results
def __call__(self, *args, **kwargs ):
#self.debug( "%s: heap at start\n%s" % (self, str(HP.heap()) ))
try: self.parseArguments( *args, **kwargs )
except:
self.error( "%s: exception in parsing" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "parsing" ) ))
# collect no data if tracker is the empty tracker
# and go straight to rendering
try:
if self.tracker.getTracks() == ["empty"]:
# is instance does not work because of module mapping
# type(Tracker.Empty) == SphinxReport.Tracker.Empty
# type(self.tracker) == Tracker.Empty
# if isinstance( self.tracker, Tracker.Empty):
result =self.renderer()
return ResultBlocks( result )
except AttributeError:
# for function trackers
pass
self.debug( "profile: started: tracker: %s" % (self.tracker))
# collecting data
try: self.collect()
except:
self.error( "%s: exception in collection" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "collection" ) ))
finally:
self.debug( "profile: finished: tracker: %s" % (self.tracker))
if len(self.data) == 0:
self.info( "%s: no data - processing complete" % self.tracker )
return None
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after collection: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# self.debug( "%s: heap after collection\n%s" % (self, str(HP.heap()) ))
# transform data
try: self.transform()
except:
self.error( "%s: exception in transformation" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "transformation" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after transformation: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# special Renderers - do not proceed
# Special renderers
if isinstance( self.renderer, Renderer.User):
results = ResultBlocks( title="main" )
results.append( self.renderer( self.data, ('') ) )
return results
elif isinstance( self.renderer, Renderer.Debug):
results = ResultBlocks( title="main" )
results.append( self.renderer( self.data, ('') ) )
return results
# self.debug( "%s: heap after transformation\n%s" % (self, str(HP.heap()) ))
# restrict
try: self.restrict()
except:
self.error( "%s: exception in restrict" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "restrict" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after restrict: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# exclude
try: self.exclude()
except:
self.error( "%s: exception in exclude" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "exclude" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after exclude: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# remove superfluous levels
try: self.prune()
except:
self.error( "%s: exception in pruning" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "pruning" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after pruning: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
# remove group plots
try: self.group()
except:
self.error( "%s: exception in grouping" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "grouping" ) ))
data_paths = DataTree.getPaths( self.data )
self.debug( "%s: after grouping: %i data_paths: %s" % (self,len(data_paths), str(data_paths)))
self.debug( "profile: started: renderer: %s" % (self.renderer))
try: result = self.render()
except:
self.error( "%s: exception in rendering" % self )
return ResultBlocks(ResultBlocks( Utils.buildException( "rendering" ) ))
finally:
self.debug( "profile: finished: renderer: %s" % (self.renderer))
#self.debug( "%s: heap at end\n%s" % (self, str(HP.heap()) ))
return result
