def getTrackers(fullpath):
"""retrieve a tracker and its associated code.
The tracker is not instantiated.
returns a tuple (code, tracker, module, flag).
The flag indicates whether that tracker is derived from the
tracker base class. If False, that tracker will not be listed
as an available tracker, though it might be specified at the
command line.
"""
name, cls = os.path.splitext(fullpath)
# remove leading '.'
cls = cls[1:]
module_name = os.path.basename(name)
module, pathname = Utils.getModule(name)
trackers = []
for name in dir(module):
obj = getattr(module, name)
try:
if Utils.isClass(obj):
trackers.append((name, obj, module_name, True))
else:
trackers.append((name, obj, module_name, False))
except ValueError:
pass
return trackers
def __call__(self, data):
if self.nlevels is None:
# do not group
return self.transform(data)
dataframes, keys = [], []
group_levels = Utils.getGroupLevels(
data,
modify_levels=self.nlevels,
)
for key, group in data.groupby(level=group_levels):
self.debug('applying transformation on group %s' % str(key))
df = self.transform(group)
if df is not None:
dataframes.append(df)
keys.append(key)
df = pandas.concat(dataframes, keys=keys)
if self.prune_dataframe:
# reset dataframe index - keep the same levels
Utils.pruneDataFrameIndex(df, original=data)
self.debug("transform: finished")
return df
def __call__(self, dataframe, path):
# convert to dataframe
# index has test names
# columns are description, info, status
columns = ('description', 'info', 'status', 'name')
if set(dataframe.columns) != set(columns):
raise ValueError("invalid columns: expected '%s', got '%s' " %
(columns, dataframe.columns))
lines = []
dirname = os.path.join(os.path.dirname(
sys.modules["CGATReport"].__file__), "images")
descriptions = {}
title = "status"
# add header
lines.append(".. csv-table:: %s" % "table")
lines.append(' :header: "Track", "Test", "", "Status", "Info"')
lines.append('')
rows = []
for index, values in dataframe.iterrows():
testname = values['name']
status = values['status']
try:
image = ".. image:: {}\n :width: 32".format(
os.path.join(dirname,
self.map_code2image[status.upper()]))
except KeyError:
image = ""
rows.append({
"test": testname,
"description": values["description"],
"info": values['info'],
"status": status,
"track": path2str(index),
"image": image,
})
descriptions[testname] = values["description"]
# filter and sort table
table = [self.columns]
table.extend([[row[x] for x in self.columns] for row in rows])
lines = Utils.table2rst(table).split("\n")
if self.display_legend:
lines.append(".. glossary::")
lines.append("")
for test, description in descriptions.items():
lines.append('%s\n%s\n' % (Utils.indent(test, 3),
Utils.indent(description, 6)))
return ResultBlocks(ResultBlock("\n".join(lines), title=""))
def __call__(self, dataframe, path):
# convert to dataframe
# index has test names
# columns are description, info, status
columns = ('description', 'info', 'status', 'name')
if set(dataframe.columns) != set(columns):
raise ValueError("invalid columns: expected '%s', got '%s' " %
(columns, dataframe.columns))
lines = []
dirname = os.path.join(os.path.dirname(
sys.modules["CGATReport"].__file__), "images")
descriptions = {}
title = "status"
# add header
lines.append(".. csv-table:: %s" % "table")
lines.append(' :header: "Track", "Test", "", "Status", "Info"')
lines.append('')
for index, values in dataframe.iterrows():
testname = values['name']
description = values['description']
info = values['info']
status = values['status']
track = path2str(index)
descriptions[testname] = description
try:
image = ".. image:: %s" %\
os.path.join(dirname, self.map_code2image[status.upper()])
except KeyError:
image = ""
lines.append(
' "%(track)s", ":term:`%(testname)s`", "%(image)s", "%(status)s", "%(info)s"' %
locals())
lines.append("")
lines.append(".. glossary::")
lines.append("")
for test, description in descriptions.items():
lines.append('%s\n%s\n' % (Utils.indent(test, 3),
Utils.indent(description, 6)))
return ResultBlocks(ResultBlock("\n".join(lines), title=""))
def collect(self, blocks, figure_key="", subfig=0):
'''collect html output from result blocks.
HTML output is written to a file and a link will be inserted at
the place holder.
'''
map_figure2text = {}
extension = "html"
for block in blocks:
if not hasattr(block, "html"):
continue
# remove special characters from filename. I think the docutils
# system removes these from links which later causes problems
# as the link does not point to the correct location of the
# file.
outname = Utils.quote_filename(
"%s_%s" % (self.template_name, block.title))
outputpath = os.path.join(
self.outdir, '%s.%s' % (outname, extension))
# save to file
outf = open(outputpath, "w")
outf.write(block.html)
outf.close()
# use absolute path
link = os.path.abspath(outputpath)
rst_output = "%(link)s" % locals()
map_figure2text["#$html %s$#" % block.title] = rst_output
return map_figure2text
def collect(self, blocks, figure_key="", subfig=0):
'''collect xls output from result blocks.
xls output is written to a file and a link will be inserted at
the place holder.
'''
map_figure2text = {}
extension = "xlsx"
for block in blocks:
if not hasattr(block, "xls"):
continue
outname = Utils.quote_filename(
"%s_%s" % (self.template_name, block.title))
outputpath = os.path.join(self.outdir, '%s.%s' %
(outname, extension))
# save to file
block.xls.save(outputpath)
# use absolute path
link = os.path.abspath(outputpath)
rst_output = ":download:`(link) </{}>`".format(link)
map_figure2text["#$xls %s$#" % block.title] = rst_output
return map_figure2text
def writeCode(class_name, code, inliner):
'''write code of class to file.
returns URI of written code.
'''
document = inliner.document.current_source
reference = class_name
# root of document tree
srcdir = setup.srcdir
# get the directory of the rst file
rstdir, rstfile = os.path.split(document)
(basedir, fname, basename, ext, outdir, codename,
notebookname) = Utils.build_paths(reference)
# path to root relative to rst
rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir)
# output code
linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename))
if code and basedir != outdir:
outfile = open(os.path.join(outdir, codename), "w")
for line in code:
outfile.write(line)
outfile.close()
return linked_codename
def writeNotebook(outfile, options, kwargs,
renderer_options,
transformer_options,
display_options,
modulename, name):
'''write a snippet to paste with the ipython notebook.
'''
cmd_options = [
'do_print = False',
'tracker="%s"' % options.tracker,
'renderer="%s"' % options.renderer,
'trackerdir="%s"' % options.trackerdir,
'workdir="%s"' % os.getcwd()]
for key, val in list(kwargs.items()) +\
list(renderer_options.items()) +\
list(transformer_options.items()):
if val is None:
cmd_options.append("%s" % key)
else:
if Utils.isString(val):
cmd_options.append('%s="%s"' % (key, val))
else:
cmd_options.append('%s=%s' % (key, val))
if options.transformers:
cmd_options.append(
"transformer=['%s']" % "','".join(options.transformers))
# no module name in tracker
params = {"tracker": "%s" % (name),
"options": ",\n".join(cmd_options)}
outfile.write(Utils.NOTEBOOK_TEXT_TEMPLATE % params)
def __call__(self, track, slice=None):
# note there are spaces behind the %(image)s directive to accomodate
# for path substitution
block = '''
.. figure:: %(image)s
:height: 300
'''
blocks = ResultBlocks()
tracks = sorted([x.asFile() for x in TRACKS])
for track in tracks:
files = glob.glob(
os.path.join(EXPORTDIR, "fastqc", "%s*_fastqc" % track))
for x, fn in enumerate(sorted(files)):
y = x + 1
image = os.path.abspath(
os.path.join(fn, "Images", "%s.png" % slice))
if not os.path.exists(image):
continue
blocks.append(ResultBlock(text=block % locals(),
title=os.path.basename(fn)))
return odict((("rst", "\n".join(Utils.layoutBlocks(
blocks,
layout="columns-2"))),))
def __call__(self, track, slice=None):
# note there are spaces behind the %(image)s directive to accomodate
# for path substitution
block = '''
.. figure:: %(image)s
:height: 300
'''
blocks = ResultBlocks()
tracks = sorted([x.asFile() for x in TRACKS])
for track in tracks:
files = glob.glob(
os.path.join(EXPORTDIR, "fastqc", "%s*_fastqc" % track))
for x, fn in enumerate(sorted(files)):
y = x + 1
image = os.path.abspath(
os.path.join(fn, "Images", "%s.png" % slice))
if not os.path.exists(image):
continue
blocks.append(
ResultBlock(text=block % locals(),
title=os.path.basename(fn)))
return odict(
(("rst", "\n".join(Utils.layoutBlocks(blocks,
layout="columns-2"))), ))
def import_image(self, filename):
"""import image into report. The image is hard-linked.
returns path to image for use in html.
"""
mangled_filename = re.sub("/", "_", filename)
filename = os.path.abspath(filename)
# directory in which to store images and thumbnails
outdir = Utils.getOutputDirectory()
image_filename = os.path.join(outdir, mangled_filename)
# filenames to use in html - must take document hierarchy
# into account
rst2srcdir = os.path.join(os.path.relpath(self.src_dir, start=self.rst_dir), outdir)
html_image_filename = os.path.join(rst2srcdir, mangled_filename)
try:
os.link(filename, image_filename)
except OSError:
# file exists
pass
return html_image_filename
def import_thumbnail(self, filename, thumbnail_size):
'''import image in *filename* as a thumbnail.
thumbnail_size is a tuple of (height, width) of the thumbnail
in pixels.
'''
mangled_filename = re.sub("/", "_", filename)
# directory in which to store images and thumbnails
outdir = Utils.getOutputDirectory()
thumb_filename = os.path.join(outdir,
"thumb-%s.png" % mangled_filename)
image = PIL.Image.open(filename)
image.thumbnail(thumbnail_size)
image.save(thumb_filename)
# filenames to use in html - must take document hierarchy
# into account
rst2srcdir = os.path.join(
os.path.relpath(self.src_dir, start=self.rst_dir),
outdir)
html_thumb_filename = os.path.join(rst2srcdir,
"thumb-%s.png" % mangled_filename)
if self.build_environment:
self.build_environment.dlfiles.add_file(self.rst_dir, html_thumb_filename)
return html_thumb_filename
def collect(self, blocks, figure_key="", subfig=0):
'''collect svg output from result blocks.
SVG output is written to a file and an image will be inserted
at the place holder.
'''
map_figure2text = {}
extension = "svg"
rst_text = '''.. figure:: /{absfn}
'''
for block in blocks:
if not hasattr(block, "svg"):
continue
# remove special characters from filename. I think the docutils
# system removes these from links which later causes problems
# as the link does not point to the correct location of the
# file.
outname = Utils.quote_filename(
"%s_%s" % (self.template_name, block.title))
outputpath = os.path.join(
self.outdir, '%s.%s' % (outname, extension))
# save to file
with open(outputpath, "w") as outf:
outf.write(block.svg)
# use absolute path
absfn = os.path.abspath(outputpath)
map_figure2text["#$svg %s$#" % block.title] = rst_text.format(
absfn=absfn)
return map_figure2text
def path2str(path):
'''convert path to printable string.'''
if path is None:
return ""
if Utils.isString(path):
return path
try:
return "/".join(map(str, path))
except:
return str(path)
def render(self, dataframe, path):
self.startPlot()
dataseries = Utils.toMultipleSeries(dataframe)
names = [path2str(x[0]) for x in dataseries]
data = [x[1] for x in dataseries]
R.boxplot(data, names=names)
return self.endPlot(dataframe, path)
def copy(src, dst):
fn = os.path.join(dest, dst, os.path.basename(src))
if os.path.exists(fn):
raise OSError("file %s already exists - not overwriting." % fn)
outfile = open(fn, "wb")
x = Utils.get_data("CGATReport", "templates/%s" % src)
if len(x) == 0:
raise ValueError('file %s is empty' % src)
outfile.write(x)
outfile.close()
def tracker2key(tracker):
'''derive cache filename from a tracker.'''
modulename = tracker.__module__
try:
# works for functions (def)
name = tracker.__name__
except AttributeError:
# works for functors (class)
name = tracker.__class__.__name__
return Utils.quote_filename(".".join((modulename, name)))
def __call__(self, track, slice=None):
blocks = ResultBlocks()
block = '''
.. figure:: %(image)s
:height: 300
'''
for image in glob.glob(os.path.join(IMAGEDIR, "*.png")):
blocks.append(ResultBlock(text=block % locals(),
title="image"))
return odict((("rst", "\n".join(Utils.layoutBlocks(blocks, layout="columns-2"))),))
def collect(self, blocks, figure_key="", subfig=0):
'''collect holoview output.
'''
map_figure2text = {}
extension = "svg"
rst_text = '''.. figure:: /{absfn}
'''
for block in blocks:
if not hasattr(block, "hv"):
continue
if self.engine == "mpl":
renderer = hv.Store.renderers['matplotlib'].instance(fig='svg')
plot = renderer.get_plot(block.hv)
rendered = renderer(plot, fmt="auto")
(data, info) = rendered
# remove special characters from filename. I think the docutils
# system removes these from links which later causes problems
# as the link does not point to the correct location of the
# file.
outname = Utils.quote_filename(
"%s_%s" % (self.template_name, block.title))
outputpath = os.path.join(
self.outdir, '%s.%s' % (outname, extension))
# save to file
with open(outputpath, "w") as outf:
outf.write(data)
# use absolute path
absfn = os.path.abspath(outputpath)
map_figure2text["#$hv %s$#" % block.title] = rst_text.format(
absfn=absfn)
elif self.engine == "bokeh":
renderer = hv.renderer("bokeh")
html = renderer.static_html(block.hv)
lines = [".. only:: html\n"] +\
[" .. raw:: html\n"] +\
[" " + x for x in html.splitlines()]
lines = "\n".join(lines)
map_figure2text["#$hv %s$#" % block.title] = lines
return map_figure2text
def setup(app):
setup.app = app
setup.config = app.config
setup.confdir = app.confdir
setup.srcdir = app.srcdir
setup.builddir = os.getcwd()
app.add_directive('report', report_directive)
# update global parameters in Utils module.
PARAMS = Utils.get_parameters()
app.add_config_value('PARAMS', collections.defaultdict(), 'env')
setup.logger = Component.get_logger()
return {'parallel_read_safe': True}
def render(self, dataframe, path):
self.startPlot()
dataseries = Utils.toMultipleSeries(dataframe)
# import pdb; pdb.set_trace()
# rframe = pandas.rpy.common.convert_to_r_dataframe(dataframe)
# R.boxplot(rframe)
names = [path2str(x[0]) for x in dataseries]
data = [x[1] for x in dataseries]
R.boxplot(data, names=names)
return self.endPlot(dataframe, path)
def getData(self, path):
"""get data for track and slice. Save data in persistent cache for
further use.
For functions, path should be an empty tuple.
"""
if path:
key = DataTree.path2str(path)
else:
key = "all"
result, fromcache = None, False
# trackers with options are not cached
if not self.nocache and not self.tracker_options:
try:
result = self.cache[key]
fromcache = True
except KeyError:
pass
except RuntimeError as msg:
raise RuntimeError(
"error when accessing key %s from cache: %s "
"- potential problem with unpickable object?" % (key, msg))
kwargs = {}
if self.tracker_options:
kwargs = Utils.parse_tracker_options(self.tracker_options)
if result is None:
try:
result = self.tracker(*path, **kwargs)
except Exception as msg:
self.warn("exception for tracker '%s', path '%s': msg=%s" %
(str(self.tracker),
DataTree.path2str(
path),
msg))
if VERBOSE:
self.warn(traceback.format_exc())
raise
# store in cache
if not self.nocache and not fromcache:
# exception - do not store data frames
# test with None fails for some reason
self.cache[key] = result
return result
def transform(self, data):
# check if data is melted:
if len(data.columns) != 1:
raise ValueError(
'transformer requires dataframe with '
'a single column, got %s' % data.columns)
column = data.columns[0]
# iterate over lowest levels to build a dictionary of
# sets
genesets = {}
nlevels = Utils.getDataFrameLevels(data)
for key, group in data.groupby(level=list(range(nlevels))):
if "background" in key and not self.background:
continue
genesets[key] = set(group[column])
values = []
if len(genesets) == 2:
a = set(genesets[list(genesets.keys())[0]])
b = set(genesets[list(genesets.keys())[1]])
values.append(("10", len(a - b)))
values.append(("01", len(b - a)))
values.append(("11", len(a & b)))
values.append(
("labels", list(map(path2str, list(genesets.keys())))))
elif len(genesets) == 3:
a = set(genesets[list(genesets.keys())[0]])
b = set(genesets[list(genesets.keys())[1]])
c = set(genesets[list(genesets.keys())[2]])
values.append(("100", len(a - b - c)))
values.append(("010", len(b - a - c)))
values.append(("001", len(c - a - b)))
values.append(("110", len((a & b) - c)))
values.append(("101", len((a & c) - b)))
values.append(("011", len((b & c) - a)))
values.append(("111", len((a & b) & c)))
values.append(
("labels", list(map(path2str, list(genesets.keys())))))
else:
raise ValueError(
"Can currently only cope with 2 or 3 way intersections")
return DataTree.listAsDataFrame(values)
def param_role(role, rawtext, text, lineno, inliner,
options={}, content=[]):
'''inserts a member variable of a tracker class in the text.'''
parts = text.split(".")
if len(parts) < 2:
msg = inliner.reporter.error(
':param: should be in class.value format '
': "%s" is invalid.' % text, line=lineno)
prb = inliner.problematic(rawtext, rawtext, msg)
return [prb], [msg]
class_name, parameter_name = ".".join(parts[:-1]), parts[-1]
try:
code, tracker, tracker_path = Utils.makeTracker(class_name)
except AttributeError:
tracker = None
if not tracker:
msg = inliner.reporter.error(
':param: can not find class '
'"%s".' % class_name, line=lineno)
prb = inliner.problematic(rawtext, rawtext, msg)
return [prb], [msg]
try:
value = getattr(tracker, parameter_name)
except AttributeError:
msg = inliner.reporter.error(
':param: can not find variable %s in '
': "%s" is invalid -tracker=%s.' %
(parameter_name, class_name, str(tracker)), line=lineno)
prb = inliner.problematic(rawtext, rawtext, msg)
return [prb], [msg]
linked_codename = writeCode(class_name, code, inliner)
node = nodes.reference(rawtext,
utils.unescape(str(value)),
refuri=linked_codename,
**options)
return [node], []
def _match(self, label, paths):
'''return True if any of paths match to label.'''
for s in paths:
if label == s:
return True
elif s.startswith("r(") and s.endswith(")"):
# collect pattern matches:
# remove r()
s = s[2:-1]
# remove flanking quotation marks
if s[0] in ('"', "'") and s[-1] in ('"', "'"):
s = s[1:-1]
rx = re.compile(s)
if not Utils.isString(label):
continue
if rx.search(label):
return True
return False
def value_role(role, rawtext, text, lineno, inliner,
options={}, content=[]):
'''insert a single value from a tracker into text.'''
class_name = text
try:
code, tracker, tracker_path = Utils.makeTracker(class_name)
except (AttributeError, ImportError):
tracker = None
if not tracker:
msg = inliner.reporter.error(
':value: can not find class '
'"%s".' % class_name, line=lineno)
prb = inliner.problematic(rawtext, rawtext, msg)
return [prb], [msg]
# Python 2/3
try:
value = str(tracker())
except TypeError as msg:
print("python 3 problem: %s: tracker=%s" % (msg, str(tracker())))
linked_codename = writeCode(class_name, code, inliner)
# Base URL mainly used by inliner.rfc_reference, so this is correct:
# ref = inliner.document.settings.rfc_base_url + inliner.rfc_url % value
# in example, but deprecated
# set_classes(options)
# node = nodes.literal(rawtext,
# utils.unescape(str(value)),
# **options)
node = nodes.reference(rawtext,
utils.unescape(str(value)),
refuri=linked_codename,
**options)
return [node], []
def cleanTrackers(rst_files, options, args):
'''instantiate trackers and get code.'''
trackers = set()
for f in rst_files:
for lineno, b in getBlocksFromRstFile(f):
trackers.add(b.mArguments[0])
ntested, ncleaned, nskipped = 0, 0, 0
for reference in trackers:
try:
code, tracker, tracker_path = report_directive.makeTracker(
reference)
except AttributeError:
# ignore missing trackers
nskipped += 1
continue
new_codehash = hashlib.md5("".join(code)).hexdigest()
(basedir, fname, basename, ext,
outdir, codename, notebookname) = Utils.build_paths(
reference)
codefilename = os.path.join(outdir, codename)
ntested += 1
if not os.path.exists(codefilename):
nskipped += 1
continue
old_codehash = hashlib.md5(
"".join(open(codefilename, "r").readlines())).hexdigest()
if new_codehash != old_codehash:
removed = clean.removeTracker(reference)
removed.extend(clean.removeText(reference))
print("code has changed for %s: %i files removed" %
(reference, len(removed)))
ncleaned += 1
print("CGATReport: %i Trackers changed (%i tested, %i skipped)" %
(ncleaned, ntested, nskipped))
def writeCode(class_name, code, inliner):
'''write code of class to file.
returns URI of written code.
'''
document = inliner.document.current_source
reference = class_name
# root of document tree
srcdir = setup.srcdir
# get the directory of the rst file
rstdir, rstfile = os.path.split(document)
(basedir, fname, basename, ext, outdir, codename,
notebookname) = Utils.build_paths(reference)
# path to root relative to rst
rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir)
# output code
linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename))
if code and basedir != outdir:
# patch, something is wrong with paths, files do not end up
# in build directory, but in current directory because outdir
# only has the last path components (_static/report_directive)
if not os.path.exists(outdir):
os.makedirs(outdir)
outfile = open(os.path.join(outdir, codename), "w")
for line in code:
outfile.write(line)
outfile.close()
return linked_codename
def __call__(self, *args, **kwargs):
try:
self.parseArguments(*args, **kwargs)
except:
self.error("%s: exception in parsing" % self)
return ResultBlocks(ResultBlocks(Utils.buildException("parsing")))
# collect no data if tracker is the empty tracker
# and go straight to rendering
try:
if self.tracker.getTracks() == ["empty"]:
# is instance does not work because of module mapping
# type(Tracker.Empty) == CGATReport.Tracker.Empty
# type(self.tracker) == Tracker.Empty
# if isinstance(self.tracker, Tracker.Empty):
result = self.renderer()
return ResultBlocks(result)
except AttributeError:
# for function trackers
pass
self.debug("profile: started: tracker: %s" % (self.tracker))
# collecting data
try:
self.collect()
except:
self.error("%s: exception in collection" % self)
return ResultBlocks(ResultBlocks(
Utils.buildException("collection")))
finally:
self.debug("profile: finished: tracker: %s" % (self.tracker))
if self.tree is None or len(self.tree) == 0:
self.info("%s: no data - processing complete" % self.tracker)
return None
data_paths = DataTree.getPaths(self.tree)
self.debug("%s: after collection: %i data_paths: %s" %
(self, len(data_paths), str(data_paths)))
# special Renderers - do not process data further but render
# directly. Note that no transformations will be applied.
if isinstance(self.renderer, Renderer.User):
results = ResultBlocks(title="main")
results.append(self.renderer(self.tree))
return results
elif isinstance(self.renderer, Renderer.Debug):
results = ResultBlocks(title="main")
results.append(self.renderer(self.tree))
return results
# merge all data to hierarchical indexed dataframe
self.data = DataTree.asDataFrame(self.tree)
self.debug("dataframe memory usage: total=%i,data=%i,index=%i,col=%i" %
(self.data.values.nbytes +
self.data.index.nbytes +
self.data.columns.nbytes,
self.data.values.nbytes,
self.data.index.nbytes,
self.data.columns.nbytes))
# if tracks are set by tracker, call tracker with dataframe
if self.indexFromTracker:
self.tracker.setIndex(self.data)
# transform data
try:
self.transform()
except:
self.error("%s: exception in transformation" % self)
return ResultBlocks(ResultBlocks(
Utils.buildException("transformation")))
# data_paths = DataTree.getPaths(self.data)
# self.debug("%s: after transformation: %i data_paths: %s" %
# (self, len(data_paths), str(data_paths)))
# restrict
try:
self.filterPaths(self.restrict_paths, mode="restrict")
except:
self.error("%s: exception in restrict" % self)
return ResultBlocks(ResultBlocks(
Utils.buildException("restrict")))
# data_paths = DataTree.getPaths(self.data)
# self.debug("%s: after restrict: %i data_paths: %s" %
# (self, len(data_paths), str(data_paths)))
# exclude
try:
self.filterPaths(self.exclude_paths, mode="exclude")
except:
self.error("%s: exception in exclude" % self)
return ResultBlocks(ResultBlocks(Utils.buildException("exclude")))
# data_paths = DataTree.getPaths(self.data)
# self.debug("%s: after exclude: %i data_paths: %s" %
# (self, len(data_paths), str(data_paths)))
# No pruning - maybe enable later as a user option
self.pruned = []
# try:
# self.prune()
# except:
# self.error("%s: exception in pruning" % self)
# return ResultBlocks(ResultBlocks(Utils.buildException("pruning")))
# data_paths = DataTree.getPaths(self.data)
# self.debug("%s: after pruning: %i data_paths: %s" %
# (self, len(data_paths), str(data_paths)))
try:
self.group()
except:
self.error("%s: exception in grouping" % self)
return ResultBlocks(ResultBlocks(Utils.buildException("grouping")))
# data_paths = DataTree.getPaths(self.data)
# self.debug("%s: after grouping: %i data_paths: %s" %
# (self, len(data_paths), str(data_paths)))
if self.renderer is not None:
self.debug("profile: started: renderer: %s" % (self.renderer))
try:
result = self.render()
except:
self.error("%s: exception in rendering" % self)
return ResultBlocks(ResultBlocks(
Utils.buildException("rendering")))
finally:
self.debug("profile: finished: renderer: %s" % (self.renderer))
else:
result = ResultBlocks(title="")
return result
def render(self):
'''supply the:class:`Renderer.Renderer` with the data to render.
The data supplied will depend on the ``groupby`` option.
returns a ResultBlocks data structure.
'''
self.debug("%s: rendering data started for %i items" %
(self,
len(self.data)))
# initiate output structure
results = ResultBlocks(title="")
dataframe = self.data
# dataframe.write_csv("test.csv")
if dataframe is None:
self.warn("%s: no data after conversion" % self)
raise ValueError("no data for renderer")
# special patch: set column names to pruned levels
# if there are no column names
if len(dataframe.columns) == len(self.pruned):
if list(dataframe.columns) == list(range(len(dataframe.columns))):
dataframe.columns = [x[1] for x in self.pruned]
nlevels = Utils.getDataFrameLevels(dataframe)
self.debug("%s: rendering data started. "
"levels=%i, group_level=%s" %
(self, nlevels,
str(self.group_level)))
if self.group_level < 0:
# no grouping for renderers that will accept
# a dataframe with any level of indices and no explicit
# grouping has been asked for.
results.append(self.renderer(dataframe, path=()))
else:
level = Utils.getGroupLevels(
dataframe,
max_level=self.group_level+1)
self.debug("%s: grouping by levels: %s" %
(self, str(level)))
for key, work in dataframe.groupby(level=level):
try:
results.append(self.renderer(work,
path=key))
except:
self.error("%s: exception in rendering" % self)
results.append(
ResultBlocks(Utils.buildException("rendering")))
if len(results) == 0:
self.warn("renderer returned no data.")
raise ValueError("renderer returned no data.")
self.debug("%s: rendering data finished with %i blocks" %
(self.tracker, len(results)))
return results
def group(self):
'''rearrange dataframe for desired grouping.
Through grouping the dataframe is rearranged such that the
level at which data will be grouped will be the first level in
hierarchical index.
If grouping by "track" is set, additional level will be added
to ensure that grouping will happen.
If grouping by "slice" is set, the first two levels will
be swopped.
The group level indicates at which level in the nested dictionary
the data will be grouped with 0 indicating that everything will
grouped together.
'''
nlevels = Utils.getDataFrameLevels(self.data)
try:
default_level = self.renderer.group_level
except AttributeError:
# User rendere that is pure functions does not
# have a group_level attribute
default_level = 0
groupby = self.groupby
if str(default_level).startswith("force"):
groupby = default_level[len("force-"):]
elif groupby == "default":
groupby = default_level
if groupby == "none":
self.group_level = nlevels - 1
elif groupby == "track":
self.group_level = 0
elif groupby == "slice":
# rearrange first two levels in data tree
if nlevels > 1:
self.data = self.data.reorder_levels(
[1, 0] + range(2, nlevels))
self.group_level = 0
elif groupby == "all":
# group everything together
self.group_level = -1
elif isinstance(groupby, int):
# get group level from Renderer
if groupby < 0:
# negative levels - subtract from lowest
# level
g = nlevels + groupby - 1
else:
g = groupby
self.group_level = g
else:
self.group_level = 0
self.debug("grouping: nlevel=%i, groupby=%s, default=%s, group=%i" %
(nlevels, groupby, str(default_level), self.group_level))
return self.data
