def writeCode(class_name, code, inliner):
'''write code of class to file.
returns URI of written code.
'''
document = inliner.document.current_source
reference = class_name
# root of document tree
srcdir = setup.srcdir
# get the directory of the rst file
rstdir, rstfile = os.path.split(document)
(basedir, fname, basename, ext, outdir, codename,
notebookname) = Utils.build_paths(reference)
# path to root relative to rst
rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir)
# output code
linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename))
if code and basedir != outdir:
outfile = open(os.path.join(outdir, codename), "w")
for line in code:
outfile.write(line)
outfile.close()
return linked_codename
def cleanTrackers(rst_files, options, args):
'''instantiate trackers and get code.'''
trackers = set()
for f in rst_files:
for lineno, b in getBlocksFromRstFile(f):
trackers.add(b.mArguments[0])
ntested, ncleaned, nskipped = 0, 0, 0
for reference in trackers:
try:
code, tracker, tracker_path = report_directive.makeTracker(
reference)
except AttributeError:
# ignore missing trackers
nskipped += 1
continue
new_codehash = hashlib.md5("".join(code)).hexdigest()
(basedir, fname, basename, ext,
outdir, codename, notebookname) = Utils.build_paths(
reference)
codefilename = os.path.join(outdir, codename)
ntested += 1
if not os.path.exists(codefilename):
nskipped += 1
continue
old_codehash = hashlib.md5(
"".join(open(codefilename, "r").readlines())).hexdigest()
if new_codehash != old_codehash:
removed = clean.removeTracker(reference)
removed.extend(clean.removeText(reference))
print("code has changed for %s: %i files removed" %
(reference, len(removed)))
ncleaned += 1
print("CGATReport: %i Trackers changed (%i tested, %i skipped)" %
(ncleaned, ntested, nskipped))
def writeCode(class_name, code, inliner):
'''write code of class to file.
returns URI of written code.
'''
document = inliner.document.current_source
reference = class_name
# root of document tree
srcdir = setup.srcdir
# get the directory of the rst file
rstdir, rstfile = os.path.split(document)
(basedir, fname, basename, ext, outdir, codename,
notebookname) = Utils.build_paths(reference)
# path to root relative to rst
rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir)
# output code
linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename))
if code and basedir != outdir:
# patch, something is wrong with paths, files do not end up
# in build directory, but in current directory because outdir
# only has the last path components (_static/report_directive)
if not os.path.exists(outdir):
os.makedirs(outdir)
outfile = open(os.path.join(outdir, codename), "w")
for line in code:
outfile.write(line)
outfile.close()
return linked_codename
def run(arguments,
options,
lineno,
content,
state_machine=None,
document=None,
srcdir=None,
builddir=None,
build_environment=None):
"""process:report: directive.
*srdir* - top level directory of rst documents
*builddir* - build directory
"""
tag = "%s:%i" % (str(document), lineno)
logger = Component.get_logger()
logger.debug("report_directive.run: profile: started: rst: %s" % tag)
# sort out the paths
# reference is used for time-stamping
tracker_name = directives.uri(arguments[0])
(basedir, fname, basename, ext, outdir,
codename, notebookname) = Utils.build_paths(tracker_name)
# get the directory of the rst file
# state_machine.document.attributes['source'])
rstdir, rstfile = os.path.split(document)
# root of document tree
if srcdir is None:
srcdir = setup.srcdir
# build directory
if builddir is None:
builddir = setup.builddir
# remove symbolic links
srcdir, builddir, rstdir = [
os.path.abspath(os.path.realpath(x)) for x in (srcdir, builddir, rstdir)]
# there are three directories:
# builddir = directory where document is built in
# (usually _build/html or similar)
# rstdir = directory where rst sources are located
# srcdir = directory from which the build process is started
# path to root relative to rst
rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir)
# path to root relative to rst
rst2builddir = os.path.join(
os.path.relpath(builddir, start=rstdir), outdir)
# path relative to source (for images)
root2builddir = os.path.join(
os.path.relpath(builddir, start=srcdir), outdir)
logger.debug(
"report_directive.run: arguments=%s, options=%s, lineno=%s, "
"content=%s, document=%s" %
(str(arguments),
str(options),
str(lineno),
str(content),
str(document)))
logger.debug(
"report_directive.run: plotdir=%s, basename=%s, ext=%s, "
"fname=%s, rstdir=%s, srcdir=%s, builddir=%s" %
(tracker_name, basename, ext, fname, rstdir, srcdir, builddir))
logger.debug(
"report_directive.run: tracker_name=%s, basedir=%s, "
"rst2src=%s, root2build=%s, outdir=%s, codename=%s" %
(tracker_name, basedir, rst2srcdir, rst2builddir, outdir, codename))
# try to create. If several processes try to create it,
# testing with `if` will not work.
try:
os.makedirs(outdir)
except OSError as msg:
pass
if not os.path.exists(outdir):
raise OSError("could not create directory %s: %s" % (outdir, msg))
########################################################
# collect options
# replace placedholders
try:
options = update_options(options)
except ValueError as msg:
logger.warn("failure while updating options: %s" % msg)
logger.debug("report_directive.run: options=%s" % (str(options),))
transformer_names = []
renderer_name = None
layout = options.get("layout", "column")
long_titles = "long-titles" in options
option_map = get_option_map()
renderer_options = select_and_delete_options(
options, option_map["render"])
transformer_options = select_and_delete_options(
options, option_map["transform"])
dispatcher_options = select_and_delete_options(
options, option_map["dispatch"])
tracker_options = select_and_delete_options(
options, option_map["tracker"], expand=["tracker"])
display_options = get_default_display_options()
display_options.update(select_and_delete_options(
options, option_map["display"]))
logger.debug("report_directive.run: renderer options: %s" %
str(renderer_options))
logger.debug("report_directive.run: transformer options: %s" %
str(transformer_options))
logger.debug("report_directive.run: dispatcher options: %s" %
str(dispatcher_options))
logger.debug("report_directive.run: tracker options: %s" %
str(tracker_options))
logger.debug("report_directive.run: display options: %s" %
str(display_options))
if "transform" in display_options:
transformer_names = display_options["transform"].split(",")
del display_options["transform"]
if "render" in display_options:
renderer_name = display_options["render"]
del display_options["render"]
########################################################
# check for missing files
if renderer_name is not None:
options_key = str(renderer_options) +\
str(transformer_options) +\
str(dispatcher_options) +\
str(tracker_options) +\
str(transformer_names) +\
re.sub("\s", "", "".join(content))
options_hash = hashlib.md5(options_key.encode()).hexdigest()[:10]
template_name = Utils.quote_filename(
Config.SEPARATOR.join((tracker_name, renderer_name,
options_hash)))
filename_text = os.path.join(outdir, "%s.txt" % (template_name))
rstname = os.path.basename(filename_text)
notebookname += options_hash
logger.debug("report_directive.run: options_hash=%s" % options_hash)
###########################################################
# check for existing files
# update strategy does not use file stamps, but checks
# for presence/absence of text element and if all figures
# mentioned in the text element are present
###########################################################
queries = [re.compile("%s/(\S+.%s)" %
(root2builddir, suffix))
for suffix in ("png", "pdf", "svg")]
logger.debug("report_directive.run: checking for changed files.")
# check if text element exists
if os.path.exists(filename_text):
with open(filename_text, "r", encoding="utf-8") as inf:
lines = [x[:-1] for x in inf]
filenames = []
# check if all figures are present
for line in lines:
for query in queries:
x = query.search(line)
if x:
filenames.extend(list(x.groups()))
filenames = [os.path.join(outdir, x) for x in filenames]
if len(filenames) == 0:
logger.info("report_directive.run: %s: redo: no files found" % tag)
else:
logger.debug(
"report_directive.run: %s: checking for %s" %
(tag, str(filenames)))
for filename in filenames:
if not os.path.exists(filename):
logger.info(
"report_directive.run: %s: redo: file %s is missing" %
(tag, filename))
break
else:
logger.info(
"report_directive.run: %s: noredo: all files are present" %
tag)
# all is present - save text and return
if lines and state_machine:
state_machine.insert_input(
lines, state_machine.input_lines.source(0))
return []
else:
logger.debug(
"report_directive.run: %s: no check performed: %s missing" %
(tag, str(filename_text)))
else:
template_name = ""
filename_text = None
collect_here = False
##########################################################
# instantiate tracker, dispatcher, renderer and transformers
# and collect output
###########################################################
try:
########################################################
# find the tracker
logger.debug(
"report_directive.run: collecting tracker %s with options %s " %
(tracker_name, tracker_options))
code, tracker, tracker_path = make_tracker(
tracker_name, (), tracker_options)
if not tracker:
logger.error(
"report_directive.run: no tracker - no output from %s " %
str(document))
raise ValueError("tracker `%s` not found" % tracker_name)
logger.debug(
"report_directive.run: collected tracker %s" % tracker_name)
tracker_id = Cache.tracker2key(tracker)
########################################################
# determine the transformer
logger.debug("report_directive.run: creating transformers")
transformers = get_transformers(
transformer_names, transformer_options)
########################################################
# determine the renderer
logger.debug("report_directive.run: creating renderer.")
if renderer_name is None:
logger.error(
"report_directive.run: no renderer - no output from %s" %
str(document))
raise ValueError("the report directive requires a renderer")
renderer = get_renderer(renderer_name, renderer_options)
try:
renderer.set_paths(rstdir, srcdir, builddir)
renderer.set_display_options(display_options)
renderer.set_build_environment(build_environment)
except AttributeError:
# User renderers will not have these methods
pass
########################################################
# write code output
linked_codename = re.sub("\\\\", "/", os.path.join(rst2builddir, codename))
if code and basedir != outdir:
if six.PY2:
with open(os.path.join(outdir, codename), "w") as outfile:
for line in code:
outfile.write(line)
else:
with open(os.path.join(outdir, codename), "w",
encoding=get_encoding()) as outfile:
for line in code:
outfile.write(line)
########################################################
# write notebook snippet
linked_notebookname = re.sub(
"\\\\", "/", os.path.join(rst2builddir, notebookname))
if basedir != outdir and tracker_id is not None:
with open(os.path.join(outdir, notebookname), "w") as outfile:
Utils.writeNoteBookEntry(outfile,
renderer=renderer_name,
tracker=tracker_name,
transformers=transformer_names,
tracker_path=tracker_path,
options=list(renderer_options.items()) +
list(tracker_options.items()) +
list(transformer_options.items()))
if filename_text is not None:
linked_rstname = re.sub(
"\\\\", "/", os.path.join(rst2builddir, rstname))
else:
linked_rstname = None
##########################################################
# Initialize collectors
links = {'code_url': linked_codename,
'rst_url': linked_rstname,
'notebook_url': linked_notebookname}
collectors = []
for name, collector in get_plugins("collect").items():
collectors.append(collector(
template_name=template_name,
outdir=outdir,
rstdir=rstdir,
builddir=builddir,
srcdir=srcdir,
content=content,
display_options=display_options,
trackerd_id=tracker_id,
links=links))
# user renderers might not have a set_collectors method
try:
collect_here = not renderer.set_collectors(collectors)
except AttributeError:
collect_here = True
########################################################
# create and call dispatcher
logger.debug("report_directive.run: creating dispatcher")
dispatcher = Dispatcher.Dispatcher(tracker,
renderer,
transformers)
# add the tracker options
dispatcher_options.update(tracker_options)
blocks = dispatcher(**dispatcher_options)
if blocks is None:
blocks = ResultBlocks(
Utils.buildWarning(
"NoData",
"tracker %s returned no Data" % str(tracker)))
code = None
tracker_id = None
except:
exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
tb = "\n".join(traceback.format_tb(exceptionTraceback))
logger.error(
"report_directive.run: exception caught at %s:%i: %s %s\n%s\n" %
(str(document), lineno,
exceptionType,
exceptionValue,
tb))
blocks = ResultBlocks(Utils.buildException("invocation"))
code = None
tracker_id = None
links = {'code_url': "",
'rst_url': "",
'notebook_url': ""}
logger.debug(
"report_directive.run: profile: started: collecting: %s" % tag)
###########################################################
# replace place holders or add text
###########################################################
# add default for text-only output
requested_urls = as_list(Utils.get_params()["report_urls"])
urls = []
if "code" in requested_urls:
urls.append(":download:`code <%(code_url)s>`" % links)
if "notebook" in requested_urls:
urls.append(":download:`nb <%(notebook_url)s>`" % links)
map_figure2text = {}
if collect_here:
for collector in collectors:
map_figure2text.update(collector.collect(blocks))
map_figure2text["default-prefix"] = ""
map_figure2text["default-suffix"] = ""
if urls and "no-links" not in display_options:
url_template = "[{}]".format(" ".join(urls))
else:
url_template = ""
map_figure2text["default-prefix"] = TEMPLATE_TEXT % locals()
blocks.updatePlaceholders(map_figure2text)
# render the output taking into account the layout
lines = Utils.layoutBlocks(blocks, layout, long_titles=long_titles)
lines.append("")
# add caption
if content and "no-caption" not in display_options:
lines.extend(['::', ''])
lines.extend([' %s' % row.strip() for row in content])
lines.append("")
# encode lines
if six.PY2:
lines = [force_encode(x, encoding="ascii", errors="replace") for x in lines]
# output rst text for this renderer
if filename_text:
if six.PY2:
with open(filename_text, "w") as outf:
outf.write("\n".join(lines))
else:
with open(filename_text, "w", encoding=get_encoding()) as outf:
outf.write("\n".join(lines))
if CGATREPORT_DEBUG:
for x, l in enumerate(lines):
try:
print(("%5i %s" % (x, l)))
except UnicodeEncodeError:
print(("line skipped - unicode error"))
pass
if len(lines) and state_machine:
state_machine.insert_input(
lines, state_machine.input_lines.source(0))
logger.debug(
"report_directive.run: profile: finished: collecting: %s" % tag)
logger.debug(
"report_directive.run: profile: finished: rst: %s:%i" %
(str(document), lineno))
return []
def run(arguments,
options,
lineno,
content,
state_machine=None,
document=None,
srcdir=None,
builddir=None):
"""process:report: directive.
*srdir* - top level directory of rst documents
*builddir* - build directory
"""
tag = "%s:%i" % (str(document), lineno)
logging.debug("report_directive.run: profile: started: rst: %s" % tag)
# sort out the paths
# reference is used for time-stamping
tracker_name = directives.uri(arguments[0])
(basedir, fname, basename, ext, outdir,
codename, notebookname) = Utils.build_paths(tracker_name)
# get the directory of the rst file
# state_machine.document.attributes['source'])
rstdir, rstfile = os.path.split(document)
# root of document tree
if srcdir is None:
srcdir = setup.srcdir
# build directory
if builddir is None:
builddir = setup.builddir
# remove symbolic links
srcdir, builddir, rstdir = [
os.path.realpath(x) for x in (srcdir, builddir, rstdir)]
# there are three directories:
# builddir = directory where document is built in
# (usually _build/html or similar)
# rstdir = directory where rst sources are located
# srcdir = directory from which the build process is started
# path to root relative to rst
rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir)
# path to root relative to rst
rst2builddir = os.path.join(
os.path.relpath(builddir, start=rstdir), outdir)
# path relative to source (for images)
root2builddir = os.path.join(
os.path.relpath(builddir, start=srcdir), outdir)
logging.debug(
"report_directive.run: arguments=%s, options=%s, lineno=%s, "
"content=%s, document=%s" %
(str(arguments),
str(options),
str(lineno),
str(content),
str(document)))
logging.debug(
"report_directive.run: plotdir=%s, basename=%s, ext=%s, "
"fname=%s, rstdir=%s, srcdir=%s, builddir=%s" %
(tracker_name, basename, ext, fname, rstdir, srcdir, builddir))
logging.debug(
"report_directive.run: tracker_name=%s, basedir=%s, "
"rst2src=%s, root2build=%s, outdir=%s, codename=%s" %
(tracker_name, basedir, rst2srcdir, rst2builddir, outdir, codename))
# try to create. If several processes try to create it,
# testing with `if` will not work.
try:
os.makedirs(outdir)
except OSError as msg:
pass
if not os.path.exists(outdir):
raise OSError("could not create directory %s: %s" % (outdir, msg))
########################################################
# collect options
# replace placedholders
try:
options = Utils.updateOptions(options)
except ValueError as msg:
logging.warn("failure while updating options: %s" % msg)
logging.debug("report_directive.run: options=%s" % (str(options),))
transformer_names = []
renderer_name = None
# get layout option
layout = options.get("layout", "column")
option_map = Component.getOptionMap()
renderer_options = Utils.selectAndDeleteOptions(
options, option_map["render"])
transformer_options = Utils.selectAndDeleteOptions(
options, option_map["transform"])
dispatcher_options = Utils.selectAndDeleteOptions(
options, option_map["dispatch"])
tracker_options = Utils.selectAndDeleteOptions(
options, option_map["tracker"])
display_options = Utils.selectAndDeleteOptions(
options, option_map["display"])
logging.debug("report_directive.run: renderer options: %s" %
str(renderer_options))
logging.debug("report_directive.run: transformer options: %s" %
str(transformer_options))
logging.debug("report_directive.run: dispatcher options: %s" %
str(dispatcher_options))
logging.debug("report_directive.run: tracker options: %s" %
str(tracker_options))
logging.debug("report_directive.run: display options: %s" %
str(display_options))
if "transform" in display_options:
transformer_names = display_options["transform"].split(",")
del display_options["transform"]
if "render" in display_options:
renderer_name = display_options["render"]
del display_options["render"]
########################################################
# check for missing files
if renderer_name is not None:
options_key = str(renderer_options) +\
str(transformer_options) +\
str(dispatcher_options) +\
str(tracker_options) +\
str(transformer_names)
options_hash = hashlib.md5(options_key.encode()).hexdigest()
template_name = Utils.quote_filename(
Config.SEPARATOR.join((tracker_name, renderer_name,
options_hash)))
filename_text = os.path.join(outdir, "%s.txt" % (template_name))
notebookname += options_hash
logging.debug("report_directive.run: options_hash=%s" % options_hash)
###########################################################
# check for existing files
# update strategy does not use file stamps, but checks
# for presence/absence of text element and if all figures
# mentioned in the text element are present
###########################################################
queries = [re.compile("%s(%s\S+.%s)" %
(root2builddir, outdir, suffix))
for suffix in ("png", "pdf", "svg")]
logging.debug("report_directive.run: checking for changed files.")
# check if text element exists
if os.path.exists(filename_text):
lines = [x[:-1] for x in open(filename_text, "r").readlines()]
filenames = []
# check if all figures are present
for line in lines:
for query in queries:
x = query.search(line)
if x:
filenames.extend(list(x.groups()))
logging.debug(
"report_directive.run: %s: checking for %s" %
(tag, str(filenames)))
for filename in filenames:
if not os.path.exists(filename):
logging.info(
"report_directive.run: %s: redo: %s missing" %
(tag, filename))
break
else:
logging.info(
"report_directive.run: %s: noredo: all files are present" %
tag)
# all is present - save text and return
if lines and state_machine:
state_machine.insert_input(
lines, state_machine.input_lines.source(0))
return []
else:
logging.debug(
"report_directive.run: %s: no check performed: %s missing" %
(tag, str(filename_text)))
else:
template_name = ""
filename_text = None
##########################################################
# Initialize collectors
collectors = []
for collector in list(Component.getPlugins("collect").values()):
collectors.append(collector())
##########################################################
# instantiate tracker, dispatcher, renderer and transformers
# and collect output
###########################################################
try:
########################################################
# find the tracker
logging.debug(
"report_directive.run: collecting tracker %s with options %s " %
(tracker_name, tracker_options))
code, tracker, tracker_path = Utils.makeTracker(
tracker_name, (), tracker_options)
if not tracker:
logging.error(
"report_directive.run: no tracker - no output from %s " %
str(document))
raise ValueError("tracker `%s` not found" % tracker_name)
logging.debug(
"report_directive.run: collected tracker %s" % tracker_name)
tracker_id = Cache.tracker2key(tracker)
########################################################
# determine the transformer
logging.debug("report_directive.run: creating transformers")
transformers = Utils.getTransformers(
transformer_names, transformer_options)
########################################################
# determine the renderer
logging.debug("report_directive.run: creating renderer.")
if renderer_name is None:
logging.error(
"report_directive.run: no renderer - no output from %s" %
str(document))
raise ValueError("the report directive requires a renderer")
renderer = Utils.getRenderer(renderer_name, renderer_options)
try:
renderer.set_paths(rstdir, srcdir, builddir)
renderer.set_display_options(display_options)
except AttributeError:
# User renderers will not have these methods
pass
########################################################
# create and call dispatcher
logging.debug("report_directive.run: creating dispatcher")
dispatcher = Dispatcher.Dispatcher(tracker,
renderer,
transformers)
# add the tracker options
dispatcher_options.update(tracker_options)
blocks = dispatcher(**dispatcher_options)
if blocks is None:
blocks = ResultBlocks(ResultBlocks(
Utils.buildWarning(
"NoData",
"tracker %s returned no Data" % str(tracker))))
code = None
tracker_id = None
except:
logging.warn(
"report_directive.run: exception caught at %s:%i - see document" %
(str(document), lineno))
blocks = ResultBlocks(ResultBlocks(
Utils.buildException("invocation")))
code = None
tracker_id = None
logging.debug(
"report_directive.run: profile: started: collecting: %s" % tag)
########################################################
# write code output
linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename))
if code and basedir != outdir:
with open(os.path.join(outdir, codename), "w") as outfile:
for line in code:
outfile.write(line)
########################################################
# write notebook snippet
linked_notebookname = re.sub(
"\\\\", "/", os.path.join(rst2srcdir, notebookname))
if basedir != outdir and tracker_id is not None:
with open(os.path.join(outdir, notebookname), "w") as outfile:
Utils.writeNoteBookEntry(outfile,
renderer=renderer_name,
tracker=tracker_name,
transformers=transformer_names,
tracker_path=tracker_path,
options=renderer_options.items() +
tracker_options.items() +
transformer_options.items())
###########################################################
# collect images
###########################################################
map_figure2text = {}
links = {'code_url': linked_codename,
'notebook_url': linked_notebookname}
try:
for collector in collectors:
map_figure2text.update(collector.collect(
blocks,
template_name,
outdir,
rstdir,
builddir,
srcdir,
content,
display_options,
tracker_id,
links=links))
except:
logging.warn("report_directive.run: exception caught while "
"collecting with %s at %s:%i - see document" %
(collector, str(document), lineno))
blocks = ResultBlocks(ResultBlocks(
Utils.buildException("collection")))
code = None
tracker_id = None
###########################################################
# replace place holders or add text
###########################################################
# add default for text-only output
urls = Utils.asList(Utils.PARAMS["report_urls"])
code_url, nb_url = "", ""
if "code" in urls:
code_url = "`code <%(code_url)s>`__" % links
if "notebook" in urls:
nb_url = '`nb <%(notebook_url)s>`__' % links
map_figure2text["default-prefix"] = TEMPLATE_TEXT % locals()
map_figure2text["default-suffix"] = ""
blocks.updatePlaceholders(map_figure2text)
# render the output taking into account the layout
lines = Utils.layoutBlocks(blocks, layout)
lines.append("")
# add caption
lines.extend(['::', ''])
if content:
lines.extend([' %s' % row.strip() for row in content])
lines.append("")
lines.append("")
# output rst text for this renderer
if filename_text:
outfile = open(filename_text, "w")
outfile.write("\n".join(lines))
outfile.close()
if CGATREPORT_DEBUG:
for x, l in enumerate(lines):
print("%5i %s" % (x, l))
if len(lines) and state_machine:
state_machine.insert_input(
lines, state_machine.input_lines.source(0))
logging.debug(
"report_directive.run: profile: finished: collecting: %s" % tag)
logging.debug(
"report_directive.run: profile: finished: rst: %s:%i" %
(str(document), lineno))
return []