Esempio n. 1
0
    def __init__(self, tracker, renderer, 
                 transformers = None ):
        '''render images using an instance of a :class:`Tracker.Tracker`
        to obtain data, an optional :class:`Transformer` to transform
        the data and a :class:`Renderer.Renderer` to render the output.
        '''
        Component.__init__(self)

        self.debug("starting dispatcher '%s': tracker='%s', renderer='%s', transformer:='%s'" % \
                  (str(self), str(tracker), str(renderer), str(transformers) ) )

        self.tracker = tracker
        # add reference to self for access to tracks
        self.tracker.dispatcher = self
        self.renderer = renderer
        self.transformers = transformers

        try:
            if tracker.cache:
                self.cache = Cache.Cache( Cache.tracker2key(tracker) )
            else:
                self.cache = {}
        except AttributeError:
            self.cache = Cache.Cache( Cache.tracker2key(tracker) )

        self.data = DataTree.DataTree()

        # Level at which to group the results of Renderers
        # None is no grouping
        # 0: group on first level ('groupby=track')
        # 1: group on second level ('groupby=slice')
        # n: group on n-th level
        self.group_level = 1
Esempio n. 2
0
    def __init__(self, tracker, renderer, transformers=None):
        """render images using an instance of a :class:`Tracker.Tracker`
        to obtain data, an optional :class:`Transformer` to transform
        the data and a :class:`Renderer.Renderer` to render the output.
        """
        Component.__init__(self)

        self.debug(
            "starting dispatcher '%s': tracker='%s', renderer='%s', transformer:='%s'"
            % (str(self), str(tracker), str(renderer), str(transformers))
        )

        self.tracker = tracker
        # add reference to self for access to tracks
        self.tracker.dispatcher = self
        self.renderer = renderer
        self.transformers = transformers

        try:
            if tracker.cache:
                self.cache = Cache.Cache(Cache.tracker2key(tracker))
            else:
                self.cache = {}
        except AttributeError:
            self.cache = Cache.Cache(Cache.tracker2key(tracker))

        self.data = DataTree.DataTree()
Esempio n. 3
0
    def GET(self, tracker):

        cache = Cache.Cache(tracker, mode="r")
        data = DataTree.fromCache(cache)
        table, row_headers, col_headers = DataTree.tree2table(data)

        return render.data_table(table, row_headers, col_headers)
    ##########################################################
    ## instantiate tracker, dispatcher, renderer and transformers
    ## and collect output
    ###########################################################
    try:
        ########################################################
        # find the tracker
        logging.debug( "report_directive.run: collecting tracker %s." % reference )
        code, tracker = Utils.makeTracker( reference )
        if not tracker: 
            logging.debug( "report_directive.run: no tracker - no output from %s " % str(document) )
            raise ValueError( "tracker `%s` not found" % reference )

        logging.debug( "report_directive.run: collected tracker." )

        tracker_id = Cache.tracker2key( tracker )

        ########################################################
        # determine the transformer
        logging.debug( "report_directive.run: creating transformers." )

        transformers = Utils.getTransformers( transformer_names, **transformer_options )

        ########################################################
        # determine the renderer
        logging.debug( "report_directive.run: creating renderer." )
        
        if renderer_name == None:
            raise ValueError("the report directive requires a renderer.")

        renderer = Utils.getRenderer( renderer_name, **renderer_options )
Esempio n. 5
0
def run(arguments, options, lineno, content, state_machine=None, document=None, srcdir=None, builddir=None):
    """process :report: directive.

    *srdir* - top level directory of rst documents
    *builddir* - build directory
    """

    tag = "%s:%i" % (str(document), lineno)

    logging.debug("report_directive.run: profile: started: rst: %s" % tag)

    # sort out the paths
    # reference is used for time-stamping
    tracker_name = directives.uri(arguments[0])

    basedir, fname, basename, ext, outdir, codename, notebookname = Utils.buildPaths(tracker_name)

    # get the directory of the rst file
    rstdir, rstfile = os.path.split(document)  # state_machine.document.attributes['source'])
    # root of document tree
    if srcdir is None:
        srcdir = setup.srcdir

    # build directory
    if builddir is None:
        builddir = setup.confdir

    # remove symbolic links
    srcdir, builddir, rstdir = [os.path.realpath(x) for x in (srcdir, builddir, rstdir)]

    # there are three directories:
    # builddir = directory where document is built in (usually _build/html or similar)
    # rstdir   = directory where rst sources are located
    # srcdir   = directory from which the build process is started

    # path to root relative to rst
    rst2srcdir = os.path.join(os.path.relpath(srcdir, start=rstdir), outdir)

    # path to root relative to rst
    rst2builddir = os.path.join(os.path.relpath(builddir, start=rstdir), outdir)

    # path relative to source (for images)
    root2builddir = os.path.join(os.path.relpath(builddir, start=srcdir), outdir)

    logging.debug(
        "report_directive.run: arguments=%s, options=%s, lineno=%s, content=%s, document=%s"
        % (str(arguments), str(options), str(lineno), str(content), str(document))
    )

    logging.debug(
        "report_directive.run: plotdir=%s, basename=%s, ext=%s, fname=%s, rstdir=%s, srcdir=%s, builddir=%s"
        % (tracker_name, basename, ext, fname, rstdir, srcdir, builddir)
    )
    logging.debug(
        "report_directive.run: tracker_name=%s, basedir=%s, rst2src=%s, root2build=%s, outdir=%s, codename=%s"
        % (tracker_name, basedir, rst2srcdir, rst2builddir, outdir, codename)
    )

    # try to create. If several processes try to create it,
    # testing with `if` will not work.
    try:
        os.makedirs(outdir)
    except OSError as msg:
        pass

    if not os.path.exists(outdir):
        raise OSError("could not create directory %s: %s" % (outdir, msg))

    ########################################################
    # collect options
    # replace placedholders
    try:
        options = Utils.updateOptions(options)
    except ValueError as msg:
        logging.warn("failure while updating options: %s" % msg)

    logging.debug("report_directive.run: options=%s" % (str(options),))

    transformer_names = []
    renderer_name = None

    # get layout option
    layout = options.get("layout", "column")

    option_map = getOptionMap()
    renderer_options = Utils.selectAndDeleteOptions(options, option_map["render"])
    transformer_options = Utils.selectAndDeleteOptions(options, option_map["transform"])
    dispatcher_options = Utils.selectAndDeleteOptions(options, option_map["dispatch"])
    tracker_options = Utils.selectAndDeleteOptions(options, option_map["tracker"])
    display_options = Utils.selectAndDeleteOptions(options, option_map["display"])

    logging.debug("report_directive.run: renderer options: %s" % str(renderer_options))
    logging.debug("report_directive.run: transformer options: %s" % str(transformer_options))
    logging.debug("report_directive.run: dispatcher options: %s" % str(dispatcher_options))
    logging.debug("report_directive.run: tracker options: %s" % str(tracker_options))
    logging.debug("report_directive.run: display options: %s" % str(display_options))

    if "transform" in display_options:
        transformer_names = display_options["transform"].split(",")
        del display_options["transform"]

    if "render" in display_options:
        renderer_name = display_options["render"]
        del display_options["render"]

    ########################################################
    # check for missing files
    if renderer_name != None:

        options_key = (
            str(renderer_options)
            + str(transformer_options)
            + str(dispatcher_options)
            + str(tracker_options)
            + str(transformer_names)
        )

        options_hash = hashlib.md5(options_key.encode()).hexdigest()

        template_name = Utils.quote_filename(Config.SEPARATOR.join((tracker_name, renderer_name, options_hash)))
        filename_text = os.path.join(outdir, "%s.txt" % (template_name))

        logging.debug("report_directive.run: options_hash=%s" % options_hash)

        ###########################################################
        # check for existing files
        # update strategy does not use file stamps, but checks
        # for presence/absence of text element and if all figures
        # mentioned in the text element are present
        ###########################################################
        queries = [re.compile("%s(%s\S+.%s)" % (root2builddir, outdir, suffix)) for suffix in ("png", "pdf")]

        logging.debug("report_directive.run: checking for changed files.")

        # check if text element exists
        if os.path.exists(filename_text):

            lines = [x[:-1] for x in open(filename_text, "r").readlines()]
            filenames = []

            # check if all figures are present
            for line in lines:
                for query in queries:
                    x = query.search(line)
                    if x:
                        filenames.extend(list(x.groups()))

            logging.debug("report_directive.run: %s: checking for %s" % (tag, str(filenames)))
            for filename in filenames:
                if not os.path.exists(filename):
                    logging.info("report_directive.run: %s: redo: %s missing" % (tag, filename))
                    break
            else:
                logging.info("report_directive.run: %s: noredo: all files are present" % tag)
                ## all is present - save text and return
                if lines and state_machine:
                    state_machine.insert_input(lines, state_machine.input_lines.source(0))
                return []
        else:
            logging.debug("report_directive.run: %s: no check performed: %s missing" % (tag, str(filename_text)))
    else:
        template_name = ""
        filename_text = None

    ##########################################################
    # Initialize collectors
    collectors = []
    for collector in list(getPlugins("collect").values()):
        collectors.append(collector())

    ##########################################################
    ## instantiate tracker, dispatcher, renderer and transformers
    ## and collect output
    ###########################################################
    try:
        ########################################################
        # find the tracker
        logging.debug("report_directive.run: collecting tracker %s with options %s " % (tracker_name, tracker_options))
        code, tracker = Utils.makeTracker(tracker_name, (), tracker_options)
        if not tracker:
            logging.error("report_directive.run: no tracker - no output from %s " % str(document))
            raise ValueError("tracker `%s` not found" % tracker_name)

        logging.debug("report_directive.run: collected tracker %s" % tracker_name)

        tracker_id = Cache.tracker2key(tracker)

        ########################################################
        # determine the transformer
        logging.debug("report_directive.run: creating transformers")

        transformers = Utils.getTransformers(transformer_names, transformer_options)

        ########################################################
        # determine the renderer
        logging.debug("report_directive.run: creating renderer.")

        if renderer_name == None:
            logging.error("report_directive.run: no renderer - no output from %s" % str(document))
            raise ValueError("the report directive requires a renderer")

        renderer = Utils.getRenderer(renderer_name, renderer_options)

        ########################################################
        # create and call dispatcher
        logging.debug("report_directive.run: creating dispatcher")

        dispatcher = Dispatcher.Dispatcher(tracker, renderer, transformers)

        # add the tracker options
        dispatcher_options.update(tracker_options)
        blocks = dispatcher(**dispatcher_options)

        if blocks == None:
            blocks = ResultBlocks(
                ResultBlocks(Utils.buildWarning("NoData", "tracker %s returned no Data" % str(tracker)))
            )
            code = None
            tracker_id = None

    except:

        logging.warn("report_directive.run: exception caught at %s:%i - see document" % (str(document), lineno))

        blocks = ResultBlocks(ResultBlocks(Utils.buildException("invocation")))
        code = None
        tracker_id = None

    logging.debug("report_directive.run: profile: started: collecting: %s" % tag)

    ########################################################
    ## write code output
    linked_codename = re.sub("\\\\", "/", os.path.join(rst2srcdir, codename))
    if code and basedir != outdir:
        with open(os.path.join(outdir, codename), "w") as outfile:
            for line in code:
                outfile.write(line)

    ########################################################
    ## write notebook snippet
    linked_notebookname = re.sub("\\\\", "/", os.path.join(rst2srcdir, notebookname))
    if basedir != outdir:
        with open(os.path.join(outdir, notebookname), "w") as outfile:
            Utils.writeNoteBookEntry(
                outfile,
                renderer=renderer_name,
                tracker=tracker_name,
                transformers=transformer_names,
                options=renderer_options.items() + tracker_options.items() + transformer_options.items(),
            )

    ###########################################################
    # collect images
    ###########################################################
    map_figure2text = {}
    try:
        for collector in collectors:
            map_figure2text.update(
                collector.collect(
                    blocks,
                    template_name,
                    outdir,
                    rstdir,
                    builddir,
                    srcdir,
                    content,
                    display_options,
                    tracker_id,
                    links={"code_url": linked_codename, "notebook_url": linked_notebookname},
                )
            )
    except:

        logging.warn(
            "report_directive.run: exception caught while collecting with %s at %s:%i - see document"
            % (collector, str(document), lineno)
        )
        blocks = ResultBlocks(ResultBlocks(Utils.buildException("collection")))
        code = None
        tracker_id = None

    ###########################################################
    # replace place holders or add text
    ###########################################################
    ## add default for text-only output
    map_figure2text["default-prefix"] = TEMPLATE_TEXT % locals()
    map_figure2text["default-suffix"] = ""
    blocks.updatePlaceholders(map_figure2text)

    ###########################################################
    ## render the output taking into account the layout
    lines = Utils.layoutBlocks(blocks, layout)

    ###########################################################
    # add caption
    lines.extend(["::", ""])
    if content:
        lines.extend(["    %s" % row.strip() for row in content])
        lines.append("")

    lines.append("")

    # output rst text for this renderer
    if filename_text:
        outfile = open(filename_text, "w")
        outfile.write("\n".join(lines))
        outfile.close()

    if SPHINXREPORT_DEBUG:
        for x, l in enumerate(lines):
            print("%5i %s" % (x, l))

    if len(lines) and state_machine:
        state_machine.insert_input(lines, state_machine.input_lines.source(0))

    logging.debug("report_directive.run: profile: finished: collecting: %s" % tag)
    logging.debug("report_directive.run: profile: finished: rst: %s:%i" % (str(document), lineno))

    return []