tracker_id = Cache.tracker2key( tracker )
########################################################
# determine the transformer
logging.debug( "report_directive.run: creating transformers." )
transformers = Utils.getTransformers( transformer_names, **transformer_options )
########################################################
# determine the renderer
logging.debug( "report_directive.run: creating renderer." )
if renderer_name == None:
raise ValueError("the report directive requires a renderer.")
renderer = Utils.getRenderer( renderer_name, **renderer_options )
########################################################
# create and call dispatcher
logging.debug( "report_directive.run: creating dispatcher" )
dispatcher = Dispatcher.Dispatcher( tracker,
renderer,
transformers )
blocks = dispatcher( **dispatcher_options )
except:
logging.warn("report_directive.run: exception caught at %s:%i - see document" % (str(document), lineno) )
def run(arguments, options, lineno, content, state_machine=None, document=None, srcdir=None, builddir=None):
"""process :report: directive.
*srdir* - top level directory of rst documents
*builddir* - build directory
"""
tag = "%s:%i" % (str(document), lineno)
logging.debug("report_directive.run: profile: started: rst: %s" % tag)
# sort out the paths
# reference is used for time-stamping
tracker_name = directives.uri(arguments[0])
basedir, fname, basename, ext, outdir, codename, notebookname = Utils.buildPaths(tracker_name)
# get the directory of the rst file
rstdir, rstfile = os.path.split(document) # state_machine.document.attributes['source'])
# root of document tree
if srcdir is None:
srcdir = setup.srcdir
# build directory
if builddir is None:
builddir = setup.confdir
# remove symbolic links
srcdir, builddir, rstdir = [os.path.realpath(x) for x in (srcdir, builddir, rstdir)]
# there are three directories:
# builddir = directory where document is built in (usually _build/html or similar)
# rstdir = directory where rst sources are located
# srcdir = directory from which the build process is started
# path to root relative to rst
rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir)
# path to root relative to rst
rst2builddir = os.path.join(os.path.relpath(builddir, start=rstdir), outdir)
# path relative to source (for images)
root2builddir = os.path.join(os.path.relpath(builddir, start=srcdir), outdir)
logging.debug(
"report_directive.run: arguments=%s, options=%s, lineno=%s, content=%s, document=%s"
% (str(arguments), str(options), str(lineno), str(content), str(document))
)
logging.debug(
"report_directive.run: plotdir=%s, basename=%s, ext=%s, fname=%s, rstdir=%s, srcdir=%s, builddir=%s"
% (tracker_name, basename, ext, fname, rstdir, srcdir, builddir)
)
logging.debug(
"report_directive.run: tracker_name=%s, basedir=%s, rst2src=%s, root2build=%s, outdir=%s, codename=%s"
% (tracker_name, basedir, rst2srcdir, rst2builddir, outdir, codename)
)
# try to create. If several processes try to create it,
# testing with `if` will not work.
try:
os.makedirs(outdir)
except OSError as msg:
pass
if not os.path.exists(outdir):
raise OSError("could not create directory %s: %s" % (outdir, msg))
########################################################
# collect options
# replace placedholders
try:
options = Utils.updateOptions(options)
except ValueError as msg:
logging.warn("failure while updating options: %s" % msg)
logging.debug("report_directive.run: options=%s" % (str(options),))
transformer_names = []
renderer_name = None
# get layout option
layout = options.get("layout", "column")
option_map = getOptionMap()
renderer_options = Utils.selectAndDeleteOptions(options, option_map["render"])
transformer_options = Utils.selectAndDeleteOptions(options, option_map["transform"])
dispatcher_options = Utils.selectAndDeleteOptions(options, option_map["dispatch"])
tracker_options = Utils.selectAndDeleteOptions(options, option_map["tracker"])
display_options = Utils.selectAndDeleteOptions(options, option_map["display"])
logging.debug("report_directive.run: renderer options: %s" % str(renderer_options))
logging.debug("report_directive.run: transformer options: %s" % str(transformer_options))
logging.debug("report_directive.run: dispatcher options: %s" % str(dispatcher_options))
logging.debug("report_directive.run: tracker options: %s" % str(tracker_options))
logging.debug("report_directive.run: display options: %s" % str(display_options))
if "transform" in display_options:
transformer_names = display_options["transform"].split(",")
del display_options["transform"]
if "render" in display_options:
renderer_name = display_options["render"]
del display_options["render"]
########################################################
# check for missing files
if renderer_name != None:
options_key = (
str(renderer_options)
+ str(transformer_options)
+ str(dispatcher_options)
+ str(tracker_options)
+ str(transformer_names)
)
options_hash = hashlib.md5(options_key.encode()).hexdigest()
template_name = Utils.quote_filename(Config.SEPARATOR.join((tracker_name, renderer_name, options_hash)))
filename_text = os.path.join(outdir, "%s.txt" % (template_name))
logging.debug("report_directive.run: options_hash=%s" % options_hash)
###########################################################
# check for existing files
# update strategy does not use file stamps, but checks
# for presence/absence of text element and if all figures
# mentioned in the text element are present
###########################################################
queries = [re.compile("%s(%s\S+.%s)" % (root2builddir, outdir, suffix)) for suffix in ("png", "pdf")]
logging.debug("report_directive.run: checking for changed files.")
# check if text element exists
if os.path.exists(filename_text):
lines = [x[:-1] for x in open(filename_text, "r").readlines()]
filenames = []
# check if all figures are present
for line in lines:
for query in queries:
x = query.search(line)
if x:
filenames.extend(list(x.groups()))
logging.debug("report_directive.run: %s: checking for %s" % (tag, str(filenames)))
for filename in filenames:
if not os.path.exists(filename):
logging.info("report_directive.run: %s: redo: %s missing" % (tag, filename))
break
else:
logging.info("report_directive.run: %s: noredo: all files are present" % tag)
## all is present - save text and return
if lines and state_machine:
state_machine.insert_input(lines, state_machine.input_lines.source(0))
return []
else:
logging.debug("report_directive.run: %s: no check performed: %s missing" % (tag, str(filename_text)))
else:
template_name = ""
filename_text = None
##########################################################
# Initialize collectors
collectors = []
for collector in list(getPlugins("collect").values()):
collectors.append(collector())
##########################################################
## instantiate tracker, dispatcher, renderer and transformers
## and collect output
###########################################################
try:
########################################################
# find the tracker
logging.debug("report_directive.run: collecting tracker %s with options %s " % (tracker_name, tracker_options))
code, tracker = Utils.makeTracker(tracker_name, (), tracker_options)
if not tracker:
logging.error("report_directive.run: no tracker - no output from %s " % str(document))
raise ValueError("tracker `%s` not found" % tracker_name)
logging.debug("report_directive.run: collected tracker %s" % tracker_name)
tracker_id = Cache.tracker2key(tracker)
########################################################
# determine the transformer
logging.debug("report_directive.run: creating transformers")
transformers = Utils.getTransformers(transformer_names, transformer_options)
########################################################
# determine the renderer
logging.debug("report_directive.run: creating renderer.")
if renderer_name == None:
logging.error("report_directive.run: no renderer - no output from %s" % str(document))
raise ValueError("the report directive requires a renderer")
renderer = Utils.getRenderer(renderer_name, renderer_options)
########################################################
# create and call dispatcher
logging.debug("report_directive.run: creating dispatcher")
dispatcher = Dispatcher.Dispatcher(tracker, renderer, transformers)
# add the tracker options
dispatcher_options.update(tracker_options)
blocks = dispatcher(**dispatcher_options)
if blocks == None:
blocks = ResultBlocks(
ResultBlocks(Utils.buildWarning("NoData", "tracker %s returned no Data" % str(tracker)))
)
code = None
tracker_id = None
except:
logging.warn("report_directive.run: exception caught at %s:%i - see document" % (str(document), lineno))
blocks = ResultBlocks(ResultBlocks(Utils.buildException("invocation")))
code = None
tracker_id = None
logging.debug("report_directive.run: profile: started: collecting: %s" % tag)
########################################################
## write code output
linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename))
if code and basedir != outdir:
with open(os.path.join(outdir, codename), "w") as outfile:
for line in code:
outfile.write(line)
########################################################
## write notebook snippet
linked_notebookname = re.sub("\\\\", "/", os.path.join(rst2srcdir, notebookname))
if basedir != outdir:
with open(os.path.join(outdir, notebookname), "w") as outfile:
Utils.writeNoteBookEntry(
outfile,
renderer=renderer_name,
tracker=tracker_name,
transformers=transformer_names,
options=renderer_options.items() + tracker_options.items() + transformer_options.items(),
)
###########################################################
# collect images
###########################################################
map_figure2text = {}
try:
for collector in collectors:
map_figure2text.update(
collector.collect(
blocks,
template_name,
outdir,
rstdir,
builddir,
srcdir,
content,
display_options,
tracker_id,
links={"code_url": linked_codename, "notebook_url": linked_notebookname},
)
)
except:
logging.warn(
"report_directive.run: exception caught while collecting with %s at %s:%i - see document"
% (collector, str(document), lineno)
)
blocks = ResultBlocks(ResultBlocks(Utils.buildException("collection")))
code = None
tracker_id = None
###########################################################
# replace place holders or add text
###########################################################
## add default for text-only output
map_figure2text["default-prefix"] = TEMPLATE_TEXT % locals()
map_figure2text["default-suffix"] = ""
blocks.updatePlaceholders(map_figure2text)
###########################################################
## render the output taking into account the layout
lines = Utils.layoutBlocks(blocks, layout)
###########################################################
# add caption
lines.extend(["::", ""])
if content:
lines.extend([" %s" % row.strip() for row in content])
lines.append("")
lines.append("")
# output rst text for this renderer
if filename_text:
outfile = open(filename_text, "w")
outfile.write("\n".join(lines))
outfile.close()
if SPHINXREPORT_DEBUG:
for x, l in enumerate(lines):
print("%5i %s" % (x, l))
if len(lines) and state_machine:
state_machine.insert_input(lines, state_machine.input_lines.source(0))
logging.debug("report_directive.run: profile: finished: collecting: %s" % tag)
logging.debug("report_directive.run: profile: finished: rst: %s:%i" % (str(document), lineno))
return []
def main():
parser = optparse.OptionParser(version="%prog version: $Id$", usage=USAGE)
parser.add_option("-t", "--tracker", dest="tracker", type="string", help="tracker to use [default=%default]")
parser.add_option("-p", "--page", dest="page", type="string", help="render an rst page [default=%default]")
parser.add_option("-a", "--tracks", dest="tracks", type="string", help="tracks to use [default=%default]")
parser.add_option(
"-m",
"--transformer",
dest="transformers",
type="string",
action="append",
help="add transformation [default=%default]",
)
parser.add_option("-s", "--slices", dest="slices", type="string", help="slices to use [default=%default]")
parser.add_option("-r", "--renderer", dest="renderer", type="string", help="renderer to use [default=%default]")
parser.add_option("-w", "--path", dest="dir_trackers", type="string", help="path to trackers [default=%default]")
parser.add_option(
"-f",
"--force",
dest="force",
action="store_true",
help="force recomputation of data by deleting cached results [default=%default]",
)
parser.add_option(
"-o",
"--option",
dest="options",
type="string",
action="append",
help="renderer options - supply as key=value pairs (without spaces). [default=%default]",
)
parser.add_option(
"--no-print",
dest="do_print",
action="store_false",
help="do not print an rst text element to create the displayed plots [default=%default].",
)
parser.add_option("--no-show", dest="do_show", action="store_false", help="do not show a plot [default=%default].")
parser.set_defaults(
tracker=None,
transformers=[],
tracks=None,
slices=None,
options=[],
renderer=None,
do_show=True,
do_print=True,
force=False,
dir_trackers=TRACKERDIR,
label="GenericLabel",
caption="add caption here",
)
(options, args) = parser.parse_args()
if len(args) == 2:
options.tracker, options.renderer = args
# configure options
options.dir_trackers = os.path.abspath(os.path.expanduser(options.dir_trackers))
if not os.path.exists(options.dir_trackers):
raise IOError("directory %s does not exist" % options.dir_trackers)
sys.path.append(options.dir_trackers)
# test plugins
kwargs = {}
for x in options.options:
if "=" in x:
data = x.split("=")
key, val = [y.strip() for y in (data[0], "=".join(data[1:]))]
else:
key, val = x.strip(), None
kwargs[key] = val
if options.tracks:
kwargs["tracks"] = options.tracks
if options.slices:
kwargs["slices"] = options.slices
if options.renderer == None:
options.renderer = "table"
kwargs = Utils.updateOptions(kwargs)
renderer = Utils.getRenderer(options.renderer, **kwargs)
transformers = Utils.getTransformers(options.transformers, **kwargs)
exclude = set(
(
"Tracker",
"TrackerSQL",
"returnLabeledData",
"returnMultipleColumnData",
"returnMultipleColumns",
"returnSingleColumn",
"returnSingleColumnData",
"SQLError",
"MultipleColumns",
"MultipleColumnData",
"LabeledData",
"DataSimple",
"Data",
)
)
if options.tracker:
trackers = []
for filename in glob.glob(os.path.join(options.dir_trackers, "*.py")):
modulename = os.path.basename(filename)
trackers.extend([x for x in getTrackers(modulename) if x[0] not in exclude])
for name, tracker, modulename, is_derived in trackers:
if name == options.tracker:
break
else:
available_trackers = set([x[0] for x in trackers if x[3]])
print "unknown tracker '%s': possible trackers are\n %s" % (
options.tracker,
"\n ".join(sorted(available_trackers)),
)
print "(the list above does not contain functions)."
sys.exit(1)
## remove everything related to that tracker for a clean slate
if options.force:
removed = SphinxReport.clean.removeTracker(name)
print "removed all data for tracker %s: %i files" % (name, len(removed))
# instantiate functors
if is_derived:
t = tracker()
# but not functions
else:
t = tracker
dispatcher = Dispatcher(t, renderer, transformers)
## needs to be resolved between renderer and dispatcher options
result = dispatcher(**kwargs)
if options.do_print:
options_rst = []
for key, val in kwargs.items():
if val == None:
options_rst.append(":%s:" % key)
else:
options_rst.append(":%s: %s" % (key, val))
print "..Template start"
print
params = {
"tracker": "%s.%s" % (modulename, name),
"renderer": options.renderer,
"label": options.label,
"options": ("\n ").join(options_rst),
"caption": options.caption,
}
if options.transformers:
params["options"] = ":transform: %s\n %s" % (",".join(options.transformers), params["options"])
print RST_TEMPLATE % params
print
print "..Template ends"
if result:
for r in result:
print "title:", r.title
for s in r:
print str(s)
if options.do_show:
if options.renderer.startswith("r-"):
print "press Ctrl-c to stop"
while 1:
pass
elif _pylab_helpers.Gcf.get_all_fig_managers() > 0:
plt.show()
elif options.page:
import build
SphinxReport.report_directive.DEBUG = True
SphinxReport.report_directive.FORCE = True
blocks = build.rst_reader(open(options.page, "r"))
for block in blocks:
build.run(((options.page, block),))
def main( argv = None, **kwargs ):
'''main function for test.py.
Long-form of command line arguments can also be supplied as kwargs.
If argv is not None, command line parsing will be performed.
'''
parser = optparse.OptionParser( version = "%prog version: $Id$",
usage = globals()["__doc__"] )
parser.add_option( "-t", "--tracker", dest="tracker", type="string",
help="tracker to use [default=%default]" )
parser.add_option( "-p", "--page", dest="page", type="string",
help="render an rst page [default=%default]" )
parser.add_option( "-a", "--tracks", dest="tracks", type="string",
help="tracks to use [default=%default]" )
parser.add_option( "-m", "--transformer", dest="transformers", type="string", action="append",
help="add transformation [default=%default]" )
parser.add_option( "-s", "--slices", dest="slices", type="string",
help="slices to use [default=%default]" )
parser.add_option( "-r", "--renderer", dest="renderer", type="string",
help="renderer to use [default=%default]" )
parser.add_option( "-w", "--path", "--trackerdir", dest="trackerdir", type="string",
help="path to trackers [default=%default]" )
parser.add_option( "-f", "--force", dest="force", action="store_true",
help="force recomputation of data by deleting cached results [default=%default]" )
parser.add_option( "-o", "--option", dest="options", type="string", action="append",
help="renderer options - supply as key=value pairs (without spaces). [default=%default]" )
parser.add_option( "-l", "--language", dest="language", type="choice",
choices = ("rst", "notebook"),
help="output language for snippet. Use ``rst`` to create a snippet to paste "
"into a sphinxreport document. Use ``notebook`` to create a snippet to paste "
"into an ipython notebook [default=%default]" )
parser.add_option( "--no-print", dest="do_print", action="store_false",
help = "do not print an rst text element to create the displayed plots [default=%default]." )
parser.add_option( "--no-show", dest="do_show", action="store_false",
help = "do not show a plot [default=%default]." )
parser.add_option( "-i", "--start-interpreter", dest="start_interpreter", action="store_true",
help = "do not render, but start python interpreter [default=%default]." )
parser.add_option( "-I", "--ii", "--start-ipython", dest="start_ipython", action="store_true",
help = "do not render, start ipython interpreter [default=%default]." )
parser.add_option( "--workdir", dest="workdir", type="string",
help="working directory - change to this directory before executing "
" [default=%default]" )
parser.add_option( "--hardcopy", dest="hardcopy", type="string",
help = "output images of plots. The parameter should contain one or more %s "
" The suffix determines the type of plot. "
" [default=%default]." )
parser.set_defaults(
loglevel = 1,
tracker=None,
transformers = [],
tracks=None,
slices=None,
options = [],
renderer = "table",
do_show = True,
do_print = True,
force = False,
trackerdir = TRACKERDIR,
label = "GenericLabel",
caption = "add caption here",
start_interpreter = False,
start_ipython = False,
language = "rst",
workdir = None,
dpi = 100 )
if argv is None and len(kwargs) == 0:
argv = sys.argv
if argv:
(options, args) = parser.parse_args( argv )
if len(args) == 2:
options.tracker, options.renderer = args
else:
(options, args) = parser.parse_args( [] )
######################################################
# set keyword arguments as options
for keyword, value in kwargs.items():
if hasattr( options, keyword ):
setattr( options, keyword, value )
del kwargs[keyword]
if options.workdir is not None:
savedir = os.getcwd()
os.chdir( options.workdir )
else:
savedir = None
######################################################
# configure options
options.trackerdir = os.path.abspath( os.path.expanduser( options.trackerdir ) )
if not os.path.exists( options.trackerdir ):
raise IOError("directory %s does not exist" % options.trackerdir )
sys.path.insert( 0, options.trackerdir )
######################################################
# test plugins
for x in options.options:
if "=" in x:
data = x.split("=")
key,val = [ y.strip() for y in (data[0], "=".join(data[1:])) ]
else:
key, val = x.strip(), None
kwargs[key] = val
if options.tracks: kwargs["tracks"] = options.tracks
if options.slices: kwargs["slices"] = options.slices
kwargs = Utils.updateOptions( kwargs )
option_map = getOptionMap()
renderer_options = Utils.selectAndDeleteOptions( kwargs, option_map["render"])
transformer_options = Utils.selectAndDeleteOptions( kwargs, option_map["transform"])
display_options = Utils.selectAndDeleteOptions( kwargs, option_map["display"])
######################################################
# decide whether to render or not
if options.renderer == "none" or options.start_interpreter or \
options.start_ipython or options.language == "notebook":
renderer = None
else:
renderer = Utils.getRenderer( options.renderer, renderer_options )
transformers = Utils.getTransformers( options.transformers, transformer_options )
exclude = set( ("Tracker",
"TrackerSQL",
"returnLabeledData",
"returnMultipleColumnData",
"returnMultipleColumns",
"returnSingleColumn",
"returnSingleColumnData",
"SQLError",
"MultipleColumns",
"MultipleColumnData",
"LabeledData",
"DataSimple",
"Data" ) )
######################################################
## build from tracker
if options.tracker:
trackers = []
for filename in glob.glob( os.path.join( options.trackerdir, "*.py" )):
modulename = os.path.basename( filename )
trackers.extend( [ x for x in getTrackers( modulename ) if x[0] not in exclude ] )
if "." in options.tracker:
parts = options.tracker.split(".")
tracker_modulename = ".".join( parts[:-1] )
tracker_name = parts[-1]
else:
tracker_modulename = None
tracker_name = options.tracker
for name, tracker, modulename, is_derived in trackers:
if name == tracker_name:
if tracker_modulename is not None:
if modulename == tracker_modulename:
break
else:
break
else:
available_trackers = set( [ x[0] for x in trackers if x[3] ] )
print("unknown tracker '%s': possible trackers are\n %s" % (options.tracker, "\n ".join( sorted(available_trackers)) ))
print("(the list above does not contain functions).")
sys.exit(1)
## remove everything related to that tracker for a clean slate
if options.force:
removed = SphinxReport.clean.removeTracker( name )
print("removed all data for tracker %s: %i files" % (name, len(removed)))
# instantiate functors
if is_derived: t = tracker( **kwargs )
# but not functions
else: t = tracker
dispatcher = Dispatcher( t, renderer, transformers )
if renderer == None:
dispatcher.parseArguments( **kwargs )
result = dispatcher.collect()
result = dispatcher.transform()
options.do_print = options.language == "notebook"
options.do_show = False
options.hardcopy = False
else:
# needs to be resolved between renderer and dispatcher options
result = dispatcher( **kwargs )
if options.do_print:
sys.stdout.write(".. Template start\n\n")
if options.language == "rst":
writeRST( sys.stdout,
options,
kwargs,
renderer_options,
transformer_options,
display_options,
modulename, name)
elif options.language == "notebook":
writeNotebook( sys.stdout,
options,
kwargs,
renderer_options,
transformer_options,
display_options,
modulename, name)
sys.stdout.write ("\n.. Template end\n")
if result and renderer != None:
for r in result:
if r.title:
print ("")
print ("title: %s" % r.title)
print ("")
for s in r:
print(str(s))
if options.hardcopy:
fig_managers = _pylab_helpers.Gcf.get_all_fig_managers()
# create all the images
for figman in fig_managers:
# create all images
figid = figman.num
outfile = re.sub( "%s", str(figid), options.hardcopy)
figman.canvas.figure.savefig( outfile, dpi=options.dpi )
if result and options.do_show:
if options.renderer.startswith("r-"):
for rr in result:
for r in rr:
if hasattr(r, 'rggplot'):
from rpy2.robjects import r as R
import rpy2.rinterface
try:
R.plot( r.rggplot)
except rpy2.rinterface.RRuntimeError, msg:
if re.search("object.*not found", str(msg)):
print '%s: available columns in dataframe=%s' % \
( msg,
R('''colnames(rframe)'''))
print("press Ctrl-c to stop")
while 1: pass
elif len(_pylab_helpers.Gcf.get_all_fig_managers()) > 0:
plt.show()
else:
for rr in result:
for r in rr:
if hasattr( r, 'xls' ):
tmpfile, outpath = tempfile.mkstemp( dir ='.', suffix = '.xlsx' )
os.close(tmpfile)
print ('saving xlsx to %s' % outpath )
r.xls.save( outpath )
