def param_role(role, rawtext, text, lineno, inliner, options={}, content=[]): '''inserts a member variable of a tracker class in the text.''' parts = text.split(".") if len(parts) < 2: msg = inliner.reporter.error( ':param: should be in class.value format ' ': "%s" is invalid.' % text, line=lineno) prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] class_name, parameter_name = ".".join(parts[:-1]), parts[-1] try: code, tracker, tracker_path = Utils.makeTracker(class_name) except AttributeError: tracker = None if not tracker: msg = inliner.reporter.error( ':param: can not find class ' '"%s".' % class_name, line=lineno) prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] try: value = getattr(tracker, parameter_name) except AttributeError: msg = inliner.reporter.error( ':param: can not find variable %s in ' ': "%s" is invalid -tracker=%s.' % (parameter_name, class_name, str(tracker)), line=lineno) prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] linked_codename = writeCode(class_name, code, inliner) node = nodes.reference(rawtext, utils.unescape(str(value)), refuri=linked_codename, **options) return [node], []
def value_role(role, rawtext, text, lineno, inliner, options={}, content=[]): '''insert a single value from a tracker into text.''' class_name = text try: code, tracker, tracker_path = Utils.makeTracker(class_name) except (AttributeError, ImportError): tracker = None if not tracker: msg = inliner.reporter.error( ':value: can not find class ' '"%s".' % class_name, line=lineno) prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] # Python 2/3 try: value = str(tracker()) except TypeError as msg: print("python 3 problem: %s: tracker=%s" % (msg, str(tracker()))) linked_codename = writeCode(class_name, code, inliner) # Base URL mainly used by inliner.rfc_reference, so this is correct: # ref = inliner.document.settings.rfc_base_url + inliner.rfc_url % value # in example, but deprecated # set_classes(options) # node = nodes.literal(rawtext, # utils.unescape(str(value)), # **options) node = nodes.reference(rawtext, utils.unescape(str(value)), refuri=linked_codename, **options) return [node], []
def main(argv=None, **kwargs): '''main function for test.py. Long-form of command line arguments can also be supplied as kwargs. If argv is not None, command line parsing will be performed. ''' parser = optparse.OptionParser(version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option("-t", "--tracker", dest="tracker", type="string", help="tracker to use [default=%default]") parser.add_option("-p", "--page", dest="page", type="string", help="render an rst page [default=%default]") parser.add_option("-a", "--tracks", dest="tracks", type="string", help="tracks to use [default=%default]") parser.add_option("-m", "--transformer", dest="transformers", type="string", action="append", help="add transformation [default=%default]") parser.add_option("-s", "--slices", dest="slices", type="string", help="slices to use [default=%default]") parser.add_option("-r", "--renderer", dest="renderer", type="string", help="renderer to use [default=%default]") parser.add_option("-w", "--path", "--trackerdir", dest="trackerdir", type="string", help="path to trackers [default=%default]") parser.add_option("-f", "--force", dest="force", action="store_true", help="force recomputation of data by deleting cached " "results [default=%default]") parser.add_option("-o", "--option", dest="options", type="string", action="append", help="renderer options - supply as key=value pairs " "(without spaces). [default=%default]") parser.add_option("-l", "--language", dest="language", type="choice", choices=("rst", "notebook"), help="output language for snippet. Use ``rst`` " "to create a snippet to paste " "into a cgatreport document. Use ``notebook`` to " "create a snippet to paste " "into an ipython notebook [default=%default]") parser.add_option("--no-print", dest="do_print", action="store_false", help="do not print an rst text element to create " "the displayed plots [default=%default].") parser.add_option("--no-show", dest="do_show", action="store_false", help="do not show a plot [default=%default].") parser.add_option("--layout", dest="layout", type="string", help="output rst with layout [default=%default].") parser.add_option("-i", "--start-interpreter", dest="start_interpreter", action="store_true", help="do not render, but start python interpreter " "[default=%default].") parser.add_option("-I", "--ii", "--start-ipython", dest="start_ipython", action="store_true", help="do not render, start ipython interpreter " "[default=%default].") parser.add_option( "--workdir", dest="workdir", type="string", help="working directory - change to this directory " "before executing " "[default=%default]") parser.add_option( "--hardcopy", dest="hardcopy", type="string", help="output images of plots. The parameter should " "contain one or more %s " "The suffix determines the type of plot. " "[default=%default].") parser.set_defaults( loglevel=1, tracker=None, transformers=[], tracks=None, slices=None, options=[], renderer="table", do_show=True, do_print=True, force=False, trackerdir=TRACKERDIR, caption="add caption here", start_interpreter=False, start_ipython=False, language="rst", workdir=None, layout=None, dpi=100) if argv is None and len(kwargs) == 0: argv = sys.argv if argv: (options, args) = parser.parse_args(argv) if len(args) == 2: options.tracker, options.renderer = args else: (options, args) = parser.parse_args([]) ###################################################### # set keyword arguments as options for keyword, value in kwargs.items(): if hasattr(options, keyword): setattr(options, keyword, value) del kwargs[keyword] # change some kwarguments if options.transformers: for keyword, value in kwargs.items(): if keyword.startswith("tf"): kwargs["tf-{}".format(keyword[2:])] = value if options.workdir is not None: savedir = os.getcwd() os.chdir(options.workdir) Utils.getParameters(sorted(glob.glob("*.ini"))) else: savedir = None ###################################################### # configure options options.trackerdir = os.path.abspath( os.path.expanduser(options.trackerdir)) if os.path.exists(options.trackerdir): sys.path.insert(0, options.trackerdir) else: L.warn("directory %s does not exist" % options.trackerdir) ###################################################### # test plugins for x in options.options: if "=" in x: data = x.split("=") key, val = [y.strip() for y in (data[0], "=".join(data[1:]))] else: key, val = x.strip(), None kwargs[key] = val if options.tracks: kwargs["tracks"] = options.tracks if options.slices: kwargs["slices"] = options.slices kwargs = Utils.updateOptions(kwargs) option_map = getOptionMap() renderer_options = Utils.selectAndDeleteOptions( kwargs, option_map["render"]) transformer_options = Utils.selectAndDeleteOptions( kwargs, option_map["transform"]) display_options = Utils.selectAndDeleteOptions( kwargs, option_map["display"]) ###################################################### # decide whether to render or not if options.renderer == "none" or options.start_interpreter or \ options.start_ipython or options.language == "notebook": renderer = None else: renderer = Utils.getRenderer(options.renderer, renderer_options) transformers = Utils.getTransformers( options.transformers, transformer_options) exclude = set(("Tracker", "TrackerSQL", "returnLabeledData", "returnMultipleColumnData", "returnMultipleColumns", "returnSingleColumn", "returnSingleColumnData", "SQLError", "MultipleColumns", "MultipleColumnData", "LabeledData", "DataSimple", "Data")) ###################################################### # build from tracker if options.tracker: if "." in options.tracker: parts = options.tracker.split(".") tracker_modulename = ".".join(parts[:-1]) tracker_name = parts[-1] else: tracker_modulename = None tracker_name = options.tracker try: _code, tracker, tracker_path = Utils.makeTracker( options.tracker, (), kwargs) except ImportError: # try to find class in module trackers = [] for filename in glob.glob( os.path.join(options.trackerdir, "*.py")): modulename = os.path.basename(filename) trackers.extend( [x for x in getTrackers(modulename) if x[0] not in exclude]) for name, tracker_class, modulename, is_derived in trackers: if name == tracker_name: if tracker_modulename is not None: if modulename == tracker_modulename: break else: tracker_modulename = modulename break else: available_trackers = set([x[0] for x in trackers if x[3]]) print( "unknown tracker '%s': possible trackers are\n %s" % (options.tracker, "\n ".join(sorted(available_trackers)))) print( "(the list above does not contain functions).") sys.exit(1) # instantiate functors if is_derived: tracker = tracker_class(**kwargs) # but not functions else: tracker = tracker_class # remove everything related to that tracker for a clean slate if options.force: removed = CGATReport.clean.removeTracker(tracker_name) print("removed all data for tracker %s: %i files" % (tracker_name, len(removed))) dispatcher = Dispatcher(tracker, renderer, transformers) if renderer is None: # dispatcher.parseArguments(**kwargs) # result = dispatcher.collect() # result = dispatcher.transform() result = dispatcher(**kwargs) options.do_print = options.language == "notebook" options.do_show = False options.hardcopy = False else: # needs to be resolved between renderer and dispatcher options result = dispatcher(**kwargs) if options.do_print: sys.stdout.write(".. ---- TEMPLATE START --------\n\n") if options.language == "rst": writeRST(sys.stdout, options, kwargs, renderer_options, transformer_options, display_options, tracker_modulename, tracker_name) elif options.language == "notebook": writeNotebook(sys.stdout, options, kwargs, renderer_options, transformer_options, display_options, tracker_modulename, tracker_name) sys.stdout.write("\n.. ---- TEMPLATE END ----------\n") sys.stdout.write("\n.. ---- OUTPUT-----------------\n") if result and renderer is not None: if options.layout is not None: lines = Utils.layoutBlocks(result, layout=options.layout) print "\n".join(lines) else: for r in result: if r.title: print ("") print ("title: %s" % r.title) print ("") for s in r: print(str(s)) if options.hardcopy: fig_managers = _pylab_helpers.Gcf.get_all_fig_managers() # create all the images for figman in fig_managers: # create all images figid = figman.num outfile = re.sub("%s", str(figid), options.hardcopy) figman.canvas.figure.savefig(outfile, dpi=options.dpi) if result and options.do_show: if options.renderer.startswith("r-"): for rr in result: for r in rr: if hasattr(r, 'rggplot'): from rpy2.robjects import r as R import rpy2.rinterface try: R.plot(r.rggplot) except rpy2.rinterface.RRuntimeError, msg: if re.search("object.*not found", str(msg)): print '%s: available columns in dataframe=%s' % \ (msg, R('''colnames(rframe)''')) print("press Ctrl-c to stop") while 1: pass elif len(_pylab_helpers.Gcf.get_all_fig_managers()) > 0: plt.show() else: for rr in result: for r in rr: if hasattr(r, 'xls'): tmpfile, outpath = tempfile.mkstemp( dir='.', suffix='.xlsx') os.close(tmpfile) print ('saving xlsx to %s' % outpath) r.xls.save(outpath) elif hasattr(r, 'bokeh'): import bokeh.plotting as bk bk.show()
def run(arguments, options, lineno, content, state_machine=None, document=None, srcdir=None, builddir=None): """process:report: directive. *srdir* - top level directory of rst documents *builddir* - build directory """ tag = "%s:%i" % (str(document), lineno) logging.debug("report_directive.run: profile: started: rst: %s" % tag) # sort out the paths # reference is used for time-stamping tracker_name = directives.uri(arguments[0]) (basedir, fname, basename, ext, outdir, codename, notebookname) = Utils.build_paths(tracker_name) # get the directory of the rst file # state_machine.document.attributes['source']) rstdir, rstfile = os.path.split(document) # root of document tree if srcdir is None: srcdir = setup.srcdir # build directory if builddir is None: builddir = setup.builddir # remove symbolic links srcdir, builddir, rstdir = [ os.path.realpath(x) for x in (srcdir, builddir, rstdir)] # there are three directories: # builddir = directory where document is built in # (usually _build/html or similar) # rstdir = directory where rst sources are located # srcdir = directory from which the build process is started # path to root relative to rst rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir) # path to root relative to rst rst2builddir = os.path.join( os.path.relpath(builddir, start=rstdir), outdir) # path relative to source (for images) root2builddir = os.path.join( os.path.relpath(builddir, start=srcdir), outdir) logging.debug( "report_directive.run: arguments=%s, options=%s, lineno=%s, " "content=%s, document=%s" % (str(arguments), str(options), str(lineno), str(content), str(document))) logging.debug( "report_directive.run: plotdir=%s, basename=%s, ext=%s, " "fname=%s, rstdir=%s, srcdir=%s, builddir=%s" % (tracker_name, basename, ext, fname, rstdir, srcdir, builddir)) logging.debug( "report_directive.run: tracker_name=%s, basedir=%s, " "rst2src=%s, root2build=%s, outdir=%s, codename=%s" % (tracker_name, basedir, rst2srcdir, rst2builddir, outdir, codename)) # try to create. If several processes try to create it, # testing with `if` will not work. try: os.makedirs(outdir) except OSError as msg: pass if not os.path.exists(outdir): raise OSError("could not create directory %s: %s" % (outdir, msg)) ######################################################## # collect options # replace placedholders try: options = Utils.updateOptions(options) except ValueError as msg: logging.warn("failure while updating options: %s" % msg) logging.debug("report_directive.run: options=%s" % (str(options),)) transformer_names = [] renderer_name = None # get layout option layout = options.get("layout", "column") option_map = Component.getOptionMap() renderer_options = Utils.selectAndDeleteOptions( options, option_map["render"]) transformer_options = Utils.selectAndDeleteOptions( options, option_map["transform"]) dispatcher_options = Utils.selectAndDeleteOptions( options, option_map["dispatch"]) tracker_options = Utils.selectAndDeleteOptions( options, option_map["tracker"]) display_options = Utils.selectAndDeleteOptions( options, option_map["display"]) logging.debug("report_directive.run: renderer options: %s" % str(renderer_options)) logging.debug("report_directive.run: transformer options: %s" % str(transformer_options)) logging.debug("report_directive.run: dispatcher options: %s" % str(dispatcher_options)) logging.debug("report_directive.run: tracker options: %s" % str(tracker_options)) logging.debug("report_directive.run: display options: %s" % str(display_options)) if "transform" in display_options: transformer_names = display_options["transform"].split(",") del display_options["transform"] if "render" in display_options: renderer_name = display_options["render"] del display_options["render"] ######################################################## # check for missing files if renderer_name is not None: options_key = str(renderer_options) +\ str(transformer_options) +\ str(dispatcher_options) +\ str(tracker_options) +\ str(transformer_names) options_hash = hashlib.md5(options_key.encode()).hexdigest() template_name = Utils.quote_filename( Config.SEPARATOR.join((tracker_name, renderer_name, options_hash))) filename_text = os.path.join(outdir, "%s.txt" % (template_name)) notebookname += options_hash logging.debug("report_directive.run: options_hash=%s" % options_hash) ########################################################### # check for existing files # update strategy does not use file stamps, but checks # for presence/absence of text element and if all figures # mentioned in the text element are present ########################################################### queries = [re.compile("%s(%s\S+.%s)" % (root2builddir, outdir, suffix)) for suffix in ("png", "pdf", "svg")] logging.debug("report_directive.run: checking for changed files.") # check if text element exists if os.path.exists(filename_text): lines = [x[:-1] for x in open(filename_text, "r").readlines()] filenames = [] # check if all figures are present for line in lines: for query in queries: x = query.search(line) if x: filenames.extend(list(x.groups())) logging.debug( "report_directive.run: %s: checking for %s" % (tag, str(filenames))) for filename in filenames: if not os.path.exists(filename): logging.info( "report_directive.run: %s: redo: %s missing" % (tag, filename)) break else: logging.info( "report_directive.run: %s: noredo: all files are present" % tag) # all is present - save text and return if lines and state_machine: state_machine.insert_input( lines, state_machine.input_lines.source(0)) return [] else: logging.debug( "report_directive.run: %s: no check performed: %s missing" % (tag, str(filename_text))) else: template_name = "" filename_text = None ########################################################## # Initialize collectors collectors = [] for collector in list(Component.getPlugins("collect").values()): collectors.append(collector()) ########################################################## # instantiate tracker, dispatcher, renderer and transformers # and collect output ########################################################### try: ######################################################## # find the tracker logging.debug( "report_directive.run: collecting tracker %s with options %s " % (tracker_name, tracker_options)) code, tracker, tracker_path = Utils.makeTracker( tracker_name, (), tracker_options) if not tracker: logging.error( "report_directive.run: no tracker - no output from %s " % str(document)) raise ValueError("tracker `%s` not found" % tracker_name) logging.debug( "report_directive.run: collected tracker %s" % tracker_name) tracker_id = Cache.tracker2key(tracker) ######################################################## # determine the transformer logging.debug("report_directive.run: creating transformers") transformers = Utils.getTransformers( transformer_names, transformer_options) ######################################################## # determine the renderer logging.debug("report_directive.run: creating renderer.") if renderer_name is None: logging.error( "report_directive.run: no renderer - no output from %s" % str(document)) raise ValueError("the report directive requires a renderer") renderer = Utils.getRenderer(renderer_name, renderer_options) try: renderer.set_paths(rstdir, srcdir, builddir) renderer.set_display_options(display_options) except AttributeError: # User renderers will not have these methods pass ######################################################## # create and call dispatcher logging.debug("report_directive.run: creating dispatcher") dispatcher = Dispatcher.Dispatcher(tracker, renderer, transformers) # add the tracker options dispatcher_options.update(tracker_options) blocks = dispatcher(**dispatcher_options) if blocks is None: blocks = ResultBlocks(ResultBlocks( Utils.buildWarning( "NoData", "tracker %s returned no Data" % str(tracker)))) code = None tracker_id = None except: logging.warn( "report_directive.run: exception caught at %s:%i - see document" % (str(document), lineno)) blocks = ResultBlocks(ResultBlocks( Utils.buildException("invocation"))) code = None tracker_id = None logging.debug( "report_directive.run: profile: started: collecting: %s" % tag) ######################################################## # write code output linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename)) if code and basedir != outdir: with open(os.path.join(outdir, codename), "w") as outfile: for line in code: outfile.write(line) ######################################################## # write notebook snippet linked_notebookname = re.sub( "\\\\", "/", os.path.join(rst2srcdir, notebookname)) if basedir != outdir and tracker_id is not None: with open(os.path.join(outdir, notebookname), "w") as outfile: Utils.writeNoteBookEntry(outfile, renderer=renderer_name, tracker=tracker_name, transformers=transformer_names, tracker_path=tracker_path, options=renderer_options.items() + tracker_options.items() + transformer_options.items()) ########################################################### # collect images ########################################################### map_figure2text = {} links = {'code_url': linked_codename, 'notebook_url': linked_notebookname} try: for collector in collectors: map_figure2text.update(collector.collect( blocks, template_name, outdir, rstdir, builddir, srcdir, content, display_options, tracker_id, links=links)) except: logging.warn("report_directive.run: exception caught while " "collecting with %s at %s:%i - see document" % (collector, str(document), lineno)) blocks = ResultBlocks(ResultBlocks( Utils.buildException("collection"))) code = None tracker_id = None ########################################################### # replace place holders or add text ########################################################### # add default for text-only output urls = Utils.asList(Utils.PARAMS["report_urls"]) code_url, nb_url = "", "" if "code" in urls: code_url = "`code <%(code_url)s>`__" % links if "notebook" in urls: nb_url = '`nb <%(notebook_url)s>`__' % links map_figure2text["default-prefix"] = TEMPLATE_TEXT % locals() map_figure2text["default-suffix"] = "" blocks.updatePlaceholders(map_figure2text) # render the output taking into account the layout lines = Utils.layoutBlocks(blocks, layout) lines.append("") # add caption lines.extend(['::', '']) if content: lines.extend([' %s' % row.strip() for row in content]) lines.append("") lines.append("") # output rst text for this renderer if filename_text: outfile = open(filename_text, "w") outfile.write("\n".join(lines)) outfile.close() if CGATREPORT_DEBUG: for x, l in enumerate(lines): print("%5i %s" % (x, l)) if len(lines) and state_machine: state_machine.insert_input( lines, state_machine.input_lines.source(0)) logging.debug( "report_directive.run: profile: finished: collecting: %s" % tag) logging.debug( "report_directive.run: profile: finished: rst: %s:%i" % (str(document), lineno)) return []