Example #1
0
 def write_data_file(self, data, fn, sort_cols=False, data_format=None):
     """ Saves raw data to a dictionary for downstream use, then redirects
     to report.write_data_file() to create the file in the report directory """
     i = 1
     base_fn = fn
     while fn in report.saved_raw_data:
         fn = '{}_{}'.format(base_fn, i)
     report.saved_raw_data[fn] = data
     util_functions.write_data_file(data, fn, sort_cols, data_format)
Example #2
0
    def write_data_file(self, data, fn, sort_cols=False, data_format=None):
        """Saves raw data to a dictionary for downstream use, then redirects
        to report.write_data_file() to create the file in the report directory"""

        # Append custom module anchor if set
        mod_cust_config = getattr(self, "mod_cust_config", {})
        if "anchor" in mod_cust_config:
            fn = "{}_{}".format(fn, mod_cust_config["anchor"])

        # Generate a unique filename if the file already exists (running module multiple times)
        i = 1
        base_fn = fn
        while fn in report.saved_raw_data:
            fn = "{}_{}".format(base_fn, i)
            i += 1

        # Save the file
        report.saved_raw_data[fn] = data
        util_functions.write_data_file(data, fn, sort_cols, data_format)
Example #3
0
def make_table(dt):
    """
    Build the HTML needed for a MultiQC table.
    :param data: MultiQC datatable object
    """

    table_id = dt.pconfig.get(
        "id", "table_{}".format("".join(random.sample(letters, 4))))
    table_id = report.save_htmlid(table_id)
    t_headers = OrderedDict()
    t_modal_headers = OrderedDict()
    t_rows = OrderedDict()
    t_rows_empty = OrderedDict()
    dt.raw_vals = defaultdict(lambda: dict())
    empty_cells = dict()
    hidden_cols = 1
    table_title = dt.pconfig.get("table_title")
    if table_title is None:
        table_title = table_id.replace("_", " ").title()

    for idx, k, header in dt.get_headers_in_order():

        rid = header["rid"]

        # Build the table header cell
        shared_key = ""
        if header.get("shared_key", None) is not None:
            shared_key = " data-shared-key={}".format(header["shared_key"])

        hide = ""
        muted = ""
        checked = ' checked="checked"'
        if header.get("hidden", False) is True:
            hide = "hidden"
            muted = " text-muted"
            checked = ""
            hidden_cols += 1

        data_attr = 'data-dmax="{}" data-dmin="{}" data-namespace="{}" {}'.format(
            header["dmax"], header["dmin"], header["namespace"], shared_key)

        cell_contents = '<span class="mqc_table_tooltip" title="{}: {}">{}</span>'.format(
            header["namespace"], header["description"], header["title"])

        t_headers[
            rid] = '<th id="header_{rid}" class="{rid} {h}" {da}>{c}</th>'.format(
                rid=rid, h=hide, da=data_attr, c=cell_contents)

        empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format(
            rid=rid, h=hide)

        # Build the modal table row
        t_modal_headers[rid] = """
        <tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);">
          <td class="sorthandle ui-sortable-handle">||</span></td>
          <td style="text-align:center;">
            <input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}">
          </td>
          <td>{name}</td>
          <td>{title}</td>
          <td>{desc}</td>
          <td>{col_id}</td>
          <td>{sk}</td>
        </tr>""".format(
            rid=rid,
            muted=muted,
            checked=checked,
            tid=table_id,
            col=header["colour"],
            name=header["namespace"],
            title=header["title"],
            desc=header["description"],
            col_id="<code>{}</code>".format(k),
            sk=header.get("shared_key", ""),
        )

        # Make a colour scale
        if header["scale"] == False:
            c_scale = None
        else:
            c_scale = mqc_colour.mqc_colour_scale(header["scale"],
                                                  header["dmin"],
                                                  header["dmax"])

        # Add the data table cells
        for (s_name, samp) in dt.data[idx].items():
            if k in samp:
                val = samp[k]
                kname = "{}_{}".format(header["namespace"], rid)
                dt.raw_vals[s_name][kname] = val

                if "modify" in header and callable(header["modify"]):
                    val = header["modify"](val)

                try:
                    dmin = header["dmin"]
                    dmax = header["dmax"]
                    percentage = ((float(val) - dmin) / (dmax - dmin)) * 100
                    percentage = min(percentage, 100)
                    percentage = max(percentage, 0)
                except (ZeroDivisionError, ValueError):
                    percentage = 0

                try:
                    valstring = str(header["format"].format(val))
                except ValueError:
                    try:
                        valstring = str(header["format"].format(float(val)))
                    except ValueError:
                        valstring = str(val)
                except:
                    valstring = str(val)

                # This is horrible, but Python locale settings are worse
                if config.thousandsSep_format is None:
                    config.thousandsSep_format = '<span class="mqc_thousandSep"></span>'
                if config.decimalPoint_format is None:
                    config.decimalPoint_format = "."
                valstring = valstring.replace(".", "DECIMAL").replace(
                    ",", "THOUSAND")
                valstring = valstring.replace(
                    "DECIMAL", config.decimalPoint_format).replace(
                        "THOUSAND", config.thousandsSep_format)

                # Percentage suffixes etc
                valstring += header.get("suffix", "")

                # Conditional formatting
                cmatches = {
                    cfck: False
                    for cfc in config.table_cond_formatting_colours
                    for cfck in cfc
                }
                # Find general rules followed by column-specific rules
                for cfk in ["all_columns", rid]:
                    if cfk in config.table_cond_formatting_rules:
                        # Loop through match types
                        for ftype in cmatches.keys():
                            # Loop through array of comparison types
                            for cmp in config.table_cond_formatting_rules[
                                    cfk].get(ftype, []):
                                try:
                                    # Each comparison should be a dict with single key: val
                                    if "s_eq" in cmp and str(
                                            cmp["s_eq"]).lower() == str(
                                                val).lower():
                                        cmatches[ftype] = True
                                    if "s_contains" in cmp and str(
                                            cmp["s_contains"]).lower() in str(
                                                val).lower():
                                        cmatches[ftype] = True
                                    if "s_ne" in cmp and str(
                                            cmp["s_ne"]).lower() != str(
                                                val).lower():
                                        cmatches[ftype] = True
                                    if "eq" in cmp and float(
                                            cmp["eq"]) == float(val):
                                        cmatches[ftype] = True
                                    if "ne" in cmp and float(
                                            cmp["ne"]) != float(val):
                                        cmatches[ftype] = True
                                    if "gt" in cmp and float(
                                            cmp["gt"]) < float(val):
                                        cmatches[ftype] = True
                                    if "lt" in cmp and float(
                                            cmp["lt"]) > float(val):
                                        cmatches[ftype] = True
                                except:
                                    logger.warning(
                                        "Not able to apply table conditional formatting to '{}' ({})"
                                        .format(val, cmp))
                # Apply HTML in order of config keys
                bgcol = None
                for cfc in config.table_cond_formatting_colours:
                    for cfck in cfc:  # should always be one, but you never know
                        if cmatches[cfck]:
                            bgcol = cfc[cfck]
                if bgcol is not None:
                    valstring = '<span class="badge" style="background-color:{}">{}</span>'.format(
                        bgcol, valstring)

                # Build HTML
                if not header["scale"]:
                    if s_name not in t_rows:
                        t_rows[s_name] = dict()
                    t_rows[s_name][
                        rid] = '<td class="{rid} {h}">{v}</td>'.format(
                            rid=rid, h=hide, v=valstring)
                else:
                    if c_scale is not None:
                        col = " background-color:{};".format(
                            c_scale.get_colour(val))
                    else:
                        col = ""
                    bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(
                        percentage, col)
                    val_html = '<span class="val">{}</span>'.format(valstring)
                    wrapper_html = '<div class="wrapper">{}{}</div>'.format(
                        bar_html, val_html)

                    if s_name not in t_rows:
                        t_rows[s_name] = dict()
                    t_rows[s_name][
                        rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(
                            rid=rid, h=hide, c=wrapper_html)

                # Is this cell hidden or empty?
                if s_name not in t_rows_empty:
                    t_rows_empty[s_name] = dict()
                t_rows_empty[s_name][rid] = header.get(
                    "hidden", False) or str(val).strip() == ""

        # Remove header if we don't have any filled cells for it
        if sum([len(rows) for rows in t_rows.values()]) == 0:
            if header.get("hidden", False) is True:
                hidden_cols -= 1
            t_headers.pop(rid, None)
            t_modal_headers.pop(rid, None)
            logger.debug("Removing header {} from table, as no data".format(k))

    #
    # Put everything together
    #

    # Buttons above the table
    html = ""
    if not config.simple_output:

        # Copy Table Button
        html += """
        <button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}">
            <span class="glyphicon glyphicon-copy"></span> Copy table
        </button>
        """.format(tid=table_id)

        # Configure Columns Button
        if len(t_headers) > 1:
            html += """
            <button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal">
                <span class="glyphicon glyphicon-th"></span> Configure Columns
            </button>
            """.format(tid=table_id)

        # Sort By Highlight button
        html += """
        <button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;">
            <span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight
        </button>
        """.format(tid=table_id)

        # Scatter Plot Button
        if len(t_headers) > 1:
            html += """
            <button type="button" class="mqc_table_makeScatter btn btn-default btn-sm" data-toggle="modal" data-target="#tableScatterModal" data-table="#{tid}">
                <span class="glyphicon glyphicon glyphicon-stats"></span> Plot
            </button>
            """.format(tid=table_id)

        # "Showing x of y columns" text
        row_visibilities = [
            all(t_rows_empty[s_name].values()) for s_name in t_rows_empty
        ]
        visible_rows = [x for x in row_visibilities if not x]

        # Visible rows
        t_showing_rows_txt = (
            'Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nvisrows}</sup>/<sub>{nrows}</sub> rows'
            .format(tid=table_id,
                    nvisrows=len(visible_rows),
                    nrows=len(t_rows)))

        # How many columns are visible?
        ncols_vis = (len(t_headers) + 1) - hidden_cols
        t_showing_cols_txt = ""
        if len(t_headers) > 1:
            t_showing_cols_txt = ' and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns'.format(
                tid=table_id, ncols_vis=ncols_vis, ncols=len(t_headers))

        # Build table header text
        html += """
        <small id="{tid}_numrows_text" class="mqc_table_numrows_text">{rows}{cols}.</small>
        """.format(tid=table_id,
                   rows=t_showing_rows_txt,
                   cols=t_showing_cols_txt)

    # Build the table itself
    collapse_class = "mqc-table-collapse" if len(
        t_rows) > 10 and config.collapse_tables else ""
    html += """
        <div id="{tid}_container" class="mqc_table_container">
            <div class="table-responsive mqc-table-responsive {cc}">
                <table id="{tid}" class="table table-condensed mqc_table" data-title="{title}">
        """.format(tid=table_id, title=table_title, cc=collapse_class)

    # Build the header row
    col1_header = dt.pconfig.get("col1_header", "Sample Name")
    html += '<thead><tr><th class="rowheader">{}</th>{}</tr></thead>'.format(
        col1_header, "".join(t_headers.values()))

    # Build the table body
    html += "<tbody>"
    t_row_keys = t_rows.keys()
    if dt.pconfig.get("sortRows") is not False:
        t_row_keys = sorted(t_row_keys)
    for s_name in t_row_keys:
        # Hide the row if all cells are empty or hidden
        row_hidden = ' style="display:none"' if all(
            t_rows_empty[s_name].values()) else ""
        html += "<tr{}>".format(row_hidden)
        # Sample name row header
        html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format(
            sn=s_name)
        for k in t_headers:
            html += t_rows[s_name].get(k, empty_cells[k])
        html += "</tr>"
    html += "</tbody></table></div>"
    if len(t_rows) > 10 and config.collapse_tables:
        html += '<div class="mqc-table-expand"><span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span></div>'
    html += "</div>"

    # Build the bootstrap modal to customise columns and order
    if not config.simple_output:
        html += """
    <!-- MultiQC Table Columns Modal -->
    <div class="modal fade" id="{tid}_configModal" tabindex="-1">
      <div class="modal-dialog modal-lg">
        <div class="modal-content">
          <div class="modal-header">
            <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
            <h4 class="modal-title">{title}: Columns</h4>
          </div>
          <div class="modal-body">
            <p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p>
            <p>
                <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button>
                <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button>
            </p>
            <table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table" data-title="{title}">
              <thead>
                <tr>
                  <th class="sorthandle" style="text-align:center;">Sort</th>
                  <th style="text-align:center;">Visible</th>
                  <th>Group</th>
                  <th>Column</th>
                  <th>Description</th>
                  <th>ID</th>
                  <th>Scale</th>
                </tr>
              </thead>
              <tbody>
                {trows}
              </tbody>
            </table>
        </div>
        <div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div>
    </div> </div> </div>""".format(tid=table_id,
                                   title=table_title,
                                   trows="".join(t_modal_headers.values()))

    # Save the raw values to a file if requested
    if dt.pconfig.get("save_file") is True:
        fn = dt.pconfig.get("raw_data_fn", "multiqc_{}".format(table_id))
        util_functions.write_data_file(dt.raw_vals, fn)
        report.saved_raw_data[fn] = dt.raw_vals

    return html
Example #4
0
def matplotlib_linegraph(plotdata, pconfig=None):
    """
    Plot a line graph with Matplot lib and return a HTML string. Either embeds a base64
    encoded image within HTML or writes the plot and links to it. Should be called by
    plot_bargraph, which properly formats the input data.
    """
    if pconfig is None:
        pconfig = {}

    # Plot group ID
    if pconfig.get("id") is None:
        pconfig["id"] = "mqc_mplplot_" + "".join(random.sample(letters, 10))

    # Sanitise plot ID and check for duplicates
    pconfig["id"] = report.save_htmlid(pconfig["id"])

    # Individual plot IDs
    pids = []
    for k in range(len(plotdata)):
        try:
            name = pconfig["data_labels"][k]["name"]
        except:
            name = k + 1
        pid = "mqc_{}_{}".format(pconfig["id"], name)
        pid = report.save_htmlid(pid, skiplint=True)
        pids.append(pid)

    html = (
        '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> '
        +
        "Flat image plot. Toolbox functions such as highlighting / hiding samples will not work "
        +
        '(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>'
    )
    html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig["id"])

    # Same defaults as HighCharts for consistency
    default_colors = [
        "#7cb5ec",
        "#434348",
        "#90ed7d",
        "#f7a35c",
        "#8085e9",
        "#f15c80",
        "#e4d354",
        "#2b908f",
        "#f45b5b",
        "#91e8e1",
    ]

    # Buttons to cycle through different datasets
    if len(plotdata) > 1 and not config.simple_output:
        html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n'
        for k, p in enumerate(plotdata):
            pid = pids[k]
            active = "active" if k == 0 else ""
            try:
                name = pconfig["data_labels"][k]["name"]
            except:
                name = k + 1
            html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format(
                a=active, pid=pid, n=name)
        html += "</div>\n\n"

    # Go through datasets creating plots
    for pidx, pdata in enumerate(plotdata):

        # Plot ID
        pid = pids[pidx]

        # Save plot data to file
        fdata = OrderedDict()
        lastcats = None
        sharedcats = True
        for d in pdata:
            fdata[d["name"]] = OrderedDict()
            for i, x in enumerate(d["data"]):
                if type(x) is list:
                    fdata[d["name"]][str(x[0])] = x[1]
                    # Check to see if all categories are the same
                    if lastcats is None:
                        lastcats = [x[0] for x in d["data"]]
                    elif lastcats != [x[0] for x in d["data"]]:
                        sharedcats = False
                else:
                    try:
                        fdata[d["name"]][pconfig["categories"][i]] = x
                    except (KeyError, IndexError):
                        fdata[d["name"]][str(i)] = x

        # Custom tsv output if the x axis varies
        if not sharedcats and config.data_format == "tsv":
            fout = ""
            for d in pdata:
                fout += "\t" + "\t".join([str(x[0]) for x in d["data"]])
                fout += "\n{}\t".format(d["name"])
                fout += "\t".join([str(x[1]) for x in d["data"]])
                fout += "\n"
            with io.open(os.path.join(config.data_dir, "{}.txt".format(pid)),
                         "w",
                         encoding="utf-8") as f:
                print(fout.encode("utf-8", "ignore").decode("utf-8"), file=f)
        else:
            util_functions.write_data_file(fdata, pid)

        # Set up figure
        fig = plt.figure(figsize=(14, 6), frameon=False)
        axes = fig.add_subplot(111)

        # Go through data series
        for idx, d in enumerate(pdata):

            # Default colour index
            cidx = idx
            while cidx >= len(default_colors):
                cidx -= len(default_colors)

            # Line style
            linestyle = "solid"
            if d.get("dashStyle", None) == "Dash":
                linestyle = "dashed"

            # Reformat data (again)
            try:
                axes.plot(
                    [x[0] for x in d["data"]],
                    [x[1] for x in d["data"]],
                    label=d["name"],
                    color=d.get("color", default_colors[cidx]),
                    linestyle=linestyle,
                    linewidth=1,
                    marker=None,
                )
            except TypeError:
                # Categorical data on x axis
                axes.plot(d["data"],
                          label=d["name"],
                          color=d.get("color", default_colors[cidx]),
                          linewidth=1,
                          marker=None)

        # Tidy up axes
        axes.tick_params(labelsize=8,
                         direction="out",
                         left=False,
                         right=False,
                         top=False,
                         bottom=False)
        axes.set_xlabel(pconfig.get("xlab", ""))
        axes.set_ylabel(pconfig.get("ylab", ""))

        # Dataset specific y label
        try:
            axes.set_ylabel(pconfig["data_labels"][pidx]["ylab"])
        except:
            pass

        # Axis limits
        default_ylimits = axes.get_ylim()
        ymin = default_ylimits[0]
        if "ymin" in pconfig:
            ymin = pconfig["ymin"]
        elif "yFloor" in pconfig:
            ymin = max(pconfig["yFloor"], default_ylimits[0])
        ymax = default_ylimits[1]
        if "ymax" in pconfig:
            ymax = pconfig["ymax"]
        elif "yCeiling" in pconfig:
            ymax = min(pconfig["yCeiling"], default_ylimits[1])
        if (ymax - ymin) < pconfig.get("yMinRange", 0):
            ymax = ymin + pconfig["yMinRange"]
        axes.set_ylim((ymin, ymax))

        # Dataset specific ymax
        try:
            axes.set_ylim((ymin, pconfig["data_labels"][pidx]["ymax"]))
        except:
            pass

        default_xlimits = axes.get_xlim()
        xmin = default_xlimits[0]
        if "xmin" in pconfig:
            xmin = pconfig["xmin"]
        elif "xFloor" in pconfig:
            xmin = max(pconfig["xFloor"], default_xlimits[0])
        xmax = default_xlimits[1]
        if "xmax" in pconfig:
            xmax = pconfig["xmax"]
        elif "xCeiling" in pconfig:
            xmax = min(pconfig["xCeiling"], default_xlimits[1])
        if (xmax - xmin) < pconfig.get("xMinRange", 0):
            xmax = xmin + pconfig["xMinRange"]
        axes.set_xlim((xmin, xmax))

        # Plot title
        if "title" in pconfig:
            plt.text(0.5,
                     1.05,
                     pconfig["title"],
                     horizontalalignment="center",
                     fontsize=16,
                     transform=axes.transAxes)
        axes.grid(True,
                  zorder=10,
                  which="both",
                  axis="y",
                  linestyle="-",
                  color="#dedede",
                  linewidth=1)

        # X axis categories, if specified
        if "categories" in pconfig:
            axes.set_xticks([i for i, v in enumerate(pconfig["categories"])])
            axes.set_xticklabels(pconfig["categories"])

        # Axis lines
        xlim = axes.get_xlim()
        axes.plot([xlim[0], xlim[1]], [0, 0],
                  linestyle="-",
                  color="#dedede",
                  linewidth=2)
        axes.set_axisbelow(True)
        axes.spines["right"].set_visible(False)
        axes.spines["top"].set_visible(False)
        axes.spines["bottom"].set_visible(False)
        axes.spines["left"].set_visible(False)

        # Background colours, if specified
        if "yPlotBands" in pconfig:
            xlim = axes.get_xlim()
            for pb in pconfig["yPlotBands"]:
                axes.barh(
                    pb["from"],
                    xlim[1],
                    height=pb["to"] - pb["from"],
                    left=xlim[0],
                    color=pb["color"],
                    linewidth=0,
                    zorder=0,
                    align="edge",
                )
        if "xPlotBands" in pconfig:
            ylim = axes.get_ylim()
            for pb in pconfig["xPlotBands"]:
                axes.bar(
                    pb["from"],
                    ylim[1],
                    width=pb["to"] - pb["from"],
                    bottom=ylim[0],
                    color=pb["color"],
                    linewidth=0,
                    zorder=0,
                    align="edge",
                )

        # Tight layout - makes sure that legend fits in and stuff
        if len(pdata) <= 15:
            axes.legend(
                loc="lower center",
                bbox_to_anchor=(0, -0.22, 1, 0.102),
                ncol=5,
                mode="expand",
                fontsize=8,
                frameon=False,
            )
            plt.tight_layout(rect=[0, 0.08, 1, 0.92])
        else:
            plt.tight_layout(rect=[0, 0, 1, 0.92])

        # Should this plot be hidden on report load?
        hidediv = ""
        if pidx > 0:
            hidediv = ' style="display:none;"'

        # Save the plot to the data directory if export is requests
        if config.export_plots:
            for fformat in config.export_plot_formats:
                # Make the directory if it doesn't already exist
                plot_dir = os.path.join(config.plots_dir, fformat)
                if not os.path.exists(plot_dir):
                    os.makedirs(plot_dir)
                # Save the plot
                plot_fn = os.path.join(plot_dir, "{}.{}".format(pid, fformat))
                fig.savefig(plot_fn, format=fformat, bbox_inches="tight")

        # Output the figure to a base64 encoded string
        if getattr(get_template_mod(), "base64_plots", True) is True:
            img_buffer = io.BytesIO()
            fig.savefig(img_buffer, format="png", bbox_inches="tight")
            b64_img = base64.b64encode(img_buffer.getvalue()).decode("utf8")
            img_buffer.close()
            html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format(
                pid, hidediv, b64_img)

        # Save to a file and link <img>
        else:
            plot_relpath = os.path.join(config.plots_dir_name, "png",
                                        "{}.png".format(pid))
            html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(
                pid, hidediv, plot_relpath)

        plt.close(fig)

    # Close wrapping div
    html += "</div>"

    report.num_mpl_plots += 1

    return html
Example #5
0
def matplotlib_bargraph (plotdata, plotsamples, pconfig=None):
    """
    Plot a bargraph with Matplot lib and return a HTML string. Either embeds a base64
    encoded image within HTML or writes the plot and links to it. Should be called by
    plot_bargraph, which properly formats the input data.
    """

    if pconfig is None:
        pconfig = {}

    # Plot group ID
    if pconfig.get('id') is None:
        pconfig['id'] = 'mqc_mplplot_'+''.join(random.sample(letters, 10))

    # Sanitise plot ID and check for duplicates
    pconfig['id'] = report.save_htmlid(pconfig['id'])

    # Individual plot IDs
    pids = []
    for k in range(len(plotdata)):
        try:
            name = pconfig['data_labels'][k]
        except:
            name = k+1
        pid = 'mqc_{}_{}'.format(pconfig['id'], name)
        pid = report.save_htmlid(pid, skiplint=True)
        pids.append(pid)

    html = '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> ' + \
          'Flat image plot. Toolbox functions such as highlighting / hiding samples will not work ' + \
          '(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>'
    html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig['id'])

    # Same defaults as HighCharts for consistency
    default_colors = ['#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9',
                      '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1']

    # Counts / Percentages Switch
    if pconfig.get('cpswitch') is not False and not config.simple_output:
        if pconfig.get('cpswitch_c_active', True) is True:
            c_active = 'active'
            p_active = ''
        else:
            c_active = ''
            p_active = 'active'
            pconfig['stacking'] = 'percent'
        c_label = pconfig.get('cpswitch_counts_label', 'Counts')
        p_label = pconfig.get('cpswitch_percent_label', 'Percentages')
        html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_setcountspcnt"> \n\
            <button class="btn btn-default btn-sm {c_a} counts">{c_l}</button> \n\
            <button class="btn btn-default btn-sm {p_a} pcnt">{p_l}</button> \n\
        </div> '.format(c_a=c_active, p_a=p_active, c_l=c_label, p_l=p_label)
        if len(plotdata) > 1:
            html += ' &nbsp; &nbsp; '

    # Buttons to cycle through different datasets
    if len(plotdata) > 1 and not config.simple_output:
        html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n'
        for k, p in enumerate(plotdata):
            pid = pids[k]
            active = 'active' if k == 0 else ''
            try:
                name = pconfig['data_labels'][k]
            except:
                name = k+1
            html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format(a=active, pid=pid, n=name)
        html += '</div>\n\n'

    # Go through datasets creating plots
    for pidx, pdata in enumerate(plotdata):

        # Save plot data to file
        fdata = {}
        for d in pdata:
            for didx, dval in enumerate(d['data']):
                s_name = plotsamples[pidx][didx]
                if s_name not in fdata:
                    fdata[s_name] = dict()
                fdata[s_name][d['name']] = dval
        util_functions.write_data_file(fdata, pids[pidx])

        # Plot percentage as well as counts
        plot_pcts = [False]
        if pconfig.get('cpswitch') is not False:
            plot_pcts = [False, True]

        # Switch out NaN for 0s so that MatPlotLib doesn't ignore stuff
        for idx, d in enumerate(pdata):
            pdata[idx]['data'] = [x if not math.isnan(x) else 0 for x in d['data'] ]

        for plot_pct in plot_pcts:

            # Plot ID
            pid = pids[pidx]
            hide_plot = False
            if plot_pct is True:
                pid = '{}_pc'.format(pid)
                if pconfig.get('cpswitch_c_active', True) is True:
                    hide_plot = True
            else:
                if pconfig.get('cpswitch_c_active', True) is not True:
                    hide_plot = True

            # Set up figure
            plt_height = len(plotsamples[pidx]) / 2.3
            plt_height = max(6, plt_height) # At least 6" tall
            plt_height = min(30, plt_height) # Cap at 30" tall
            bar_width = 0.8

            fig = plt.figure(figsize=(14, plt_height), frameon=False)
            axes = fig.add_subplot(111)
            y_ind = range(len(plotsamples[pidx]))

            # Count totals for each sample
            if plot_pct is True:
                s_totals = [0 for _ in pdata[0]['data']]
                for series_idx, d in enumerate(pdata):
                    for sample_idx, v in enumerate(d['data']):
                        s_totals[sample_idx] += v

            # Plot bars
            dlabels = []
            for idx, d in enumerate(pdata):
                # Plot percentages
                values = d['data']
                if len(values) < len(y_ind):
                    values.extend([0] * (len(y_ind) - len(values)))
                if plot_pct is True:
                    for (key,var) in enumerate(values):
                        s_total = s_totals[key]
                        if s_total == 0:
                            values[key] = 0
                        else:
                            values[key] = (float(var+0.0)/float(s_total))*100

                # Get offset for stacked bars
                if idx == 0:
                    prevdata = [0] * len(plotsamples[pidx])
                else:
                    for i, p in enumerate(prevdata):
                        prevdata[i] += pdata[idx-1]['data'][i]
                # Default colour index
                cidx = idx
                while cidx >= len(default_colors):
                    cidx -= len(default_colors)
                # Save the name of this series
                dlabels.append(d['name'])
                # Add the series of bars to the plot
                axes.barh(
                    y_ind,
                    values,
                    bar_width,
                    left = prevdata,
                    color = d.get('color', default_colors[cidx]),
                    align = 'center',
                    linewidth = pconfig.get('borderWidth', 0)
                )

            # Tidy up axes
            axes.tick_params(labelsize=8, direction='out', left=False, right=False, top=False, bottom=False)
            axes.set_xlabel(pconfig.get('ylab', '')) # I know, I should fix the fact that the config is switched
            axes.set_ylabel(pconfig.get('xlab', ''))
            axes.set_yticks(y_ind) # Specify where to put the labels
            axes.set_yticklabels(plotsamples[pidx]) # Set y axis sample name labels
            axes.set_ylim((-0.5, len(y_ind)-0.5)) # Reduce padding around plot area
            if plot_pct is True:
                axes.set_xlim((0, 100))
                # Add percent symbols
                vals = axes.get_xticks()
                axes.set_xticklabels(['{:.0f}%'.format(x) for x in vals])
            else:
                default_xlimits = axes.get_xlim()
                axes.set_xlim((pconfig.get('ymin', default_xlimits[0]),pconfig.get('ymax', default_xlimits[1])))
            if 'title' in pconfig:
                top_gap = 1 + (0.5 / plt_height)
                plt.text(0.5, top_gap, pconfig['title'], horizontalalignment='center', fontsize=16, transform=axes.transAxes)
            axes.grid(True, zorder=0, which='both', axis='x', linestyle='-', color='#dedede', linewidth=1)
            axes.set_axisbelow(True)
            axes.spines['right'].set_visible(False)
            axes.spines['top'].set_visible(False)
            axes.spines['bottom'].set_visible(False)
            axes.spines['left'].set_visible(False)
            plt.gca().invert_yaxis() # y axis is reverse sorted otherwise

            # Hide some labels if we have a lot of samples
            show_nth = max(1, math.ceil(len(pdata[0]['data'])/150))
            for idx, label in enumerate(axes.get_yticklabels()):
                if idx % show_nth != 0:
                    label.set_visible(False)

            # Legend
            bottom_gap = -1 * (1 - ((plt_height - 1.5) / plt_height))
            lgd = axes.legend(dlabels, loc='lower center', bbox_to_anchor=(0, bottom_gap, 1, .102), ncol=5, mode='expand', fontsize=8, frameon=False)

            # Should this plot be hidden on report load?
            hidediv = ''
            if pidx > 0 or hide_plot:
                hidediv = ' style="display:none;"'

            # Save the plot to the data directory if export is requested
            if config.export_plots:
                for fformat in config.export_plot_formats:
                    # Make the directory if it doesn't already exist
                    plot_dir = os.path.join(config.plots_dir, fformat)
                    if not os.path.exists(plot_dir):
                        os.makedirs(plot_dir)
                    # Save the plot
                    plot_fn = os.path.join(plot_dir, '{}.{}'.format(pid, fformat))
                    fig.savefig(plot_fn, format=fformat, bbox_extra_artists=(lgd,), bbox_inches='tight')

            # Output the figure to a base64 encoded string
            if getattr(get_template_mod(), 'base64_plots', True) is True:
                img_buffer = io.BytesIO()
                fig.savefig(img_buffer, format='png', bbox_inches='tight')
                b64_img = base64.b64encode(img_buffer.getvalue()).decode('utf8')
                img_buffer.close()
                html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format(pid, hidediv, b64_img)

            # Link to the saved image
            else:
                plot_relpath = os.path.join(config.plots_dir_name, 'png', '{}.png'.format(pid))
                html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(pid, hidediv, plot_relpath)

            plt.close(fig)


    # Close wrapping div
    html += '</div>'

    report.num_mpl_plots += 1

    return html
Example #6
0
def make_table(dt, hide_bar=None):
    """
    Build the HTML needed for a MultiQC table.
    :param data: MultiQC datatable object
    """

    table_id = dt.pconfig.get(
        'id', 'table_{}'.format(''.join(random.sample(letters, 4))))
    table_id = report.save_htmlid(table_id)
    t_headers = OrderedDict()
    t_modal_headers = OrderedDict()
    t_rows = OrderedDict()
    t_rows_empty = OrderedDict()
    dt.raw_vals = defaultdict(lambda: dict())
    empty_cells = dict()
    hidden_cols = 1
    table_title = dt.pconfig.get('table_title')
    if table_title is None:
        table_title = table_id.replace("_", " ").title()

    for idx, k, header in dt.get_headers_in_order():

        rid = header['rid']

        # Build the table header cell
        shared_key = ''
        if header.get('shared_key', None) is not None:
            shared_key = ' data-shared-key={}'.format(header['shared_key'])

        hide = ''
        muted = ''
        checked = ' checked="checked"'
        if header.get('hidden', False) is True:
            hide = 'hidden'
            muted = ' text-muted'
            checked = ''
            hidden_cols += 1

        data_attr = 'data-dmax="{}" data-dmin="{}" data-namespace="{}" {}' \
            .format(header['dmax'], header['dmin'], header['namespace'], shared_key)

        if header.get('namespace'):
            cell_contents = '<span class="mqc_table_tooltip" title="{}: {}">{}</span>' \
                .format(header['namespace'], header['description'], header['title'])
        else:
            cell_contents = '<span class="mqc_table_tooltip" title="{} {}">{}</span>' \
                .format(header['namespace'], header['description'], header['title'])

        t_headers[rid] = '<th id="header_{rid}" class="{rid} {h}" {da}>{c}</th>' \
            .format(rid=rid, h=hide, da=data_attr, c=cell_contents)

        empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format(
            rid=rid, h=hide)

        # Build the modal table row
        t_modal_headers[rid] = """
        <tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);">
          <td class="sorthandle ui-sortable-handle">||</span></td>
          <td style="text-align:center;">
            <input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}">
          </td>
          <td>{name}</td>
          <td>{title}</td>
          <td>{desc}</td>
          <td>{col_id}</td>
          <td>{sk}</td>
        </tr>""".format(rid=rid,
                        muted=muted,
                        checked=checked,
                        tid=table_id,
                        col=header['colour'],
                        name=header['namespace'],
                        title=header['title'],
                        desc=header['description'],
                        col_id='<code>{}</code>'.format(k),
                        sk=header.get('shared_key', ''))

        # Make a colour scale
        if header['scale'] == False:
            c_scale = None
        else:
            c_scale = mqc_colour.mqc_colour_scale(header['scale'],
                                                  header['dmin'],
                                                  header['dmax'])

        # Add the data table cells
        for (s_name, samp) in dt.data[idx].items():
            if k in samp:
                val = samp[k]
                kname = '{}_{}'.format(header['namespace'], rid)
                dt.raw_vals[s_name][kname] = val

                # if "is_int" in header.keys():
                #     val = int(val)
                #     print(val)

                if 'modify' in header and callable(header['modify']):
                    val = header['modify'](val)

                try:
                    dmin = header['dmin']
                    dmax = header['dmax']
                    percentage = ((float(val) - dmin) / (dmax - dmin)) * 100
                    percentage = min(percentage, 100)
                    percentage = max(percentage, 0)
                except (ZeroDivisionError, ValueError):
                    percentage = 0

                try:
                    if not header.get("is_int"):
                        valstring = str(header['format'].format(val))
                    else:
                        valstring = str(int(val))

                except ValueError:
                    try:
                        valstring = str(header['format'].format(float(val)))

                    except ValueError:
                        valstring = str(val)

                except:
                    valstring = str(val)

                # This is horrible, but Python locale settings are worse
                if config.thousandsSep_format is None:
                    config.thousandsSep_format = '<span class="mqc_thousandSep"></span>'
                if config.decimalPoint_format is None:
                    config.decimalPoint_format = '.'
                valstring = valstring.replace('.', 'DECIMAL').replace(
                    ',', 'THOUSAND')
                valstring = valstring.replace(
                    'DECIMAL', config.decimalPoint_format).replace(
                        'THOUSAND', config.thousandsSep_format)

                # Percentage suffixes etc
                if header.get('suffix') == "show_perc":
                    suff_dict = header.get("suffix_dict")
                    valstring += str(suff_dict.get(s_name))
                else:
                    valstring += header.get('suffix', '')

                # Conditional formatting
                cmatches = {
                    cfck: False
                    for cfc in config.table_cond_formatting_colours
                    for cfck in cfc
                }
                # Find general rules followed by column-specific rules
                for cfk in ['all_columns', rid]:
                    if cfk in config.table_cond_formatting_rules:
                        # Loop through match types
                        for ftype in cmatches.keys():
                            # Loop through array of comparison types
                            for cmp in config.table_cond_formatting_rules[
                                    cfk].get(ftype, []):
                                try:
                                    # Each comparison should be a dict with single key: val
                                    if 's_eq' in cmp and str(
                                            cmp['s_eq']).lower() == str(
                                                val).lower():
                                        cmatches[ftype] = True
                                    if 's_contains' in cmp and str(
                                            cmp['s_contains']).lower() in str(
                                                val).lower():
                                        cmatches[ftype] = True
                                    if 's_ne' in cmp and str(
                                            cmp['s_ne']).lower() != str(
                                                val).lower():
                                        cmatches[ftype] = True
                                    if 'eq' in cmp and float(
                                            cmp['eq']) == float(val):
                                        cmatches[ftype] = True
                                    if 'ne' in cmp and float(
                                            cmp['ne']) != float(val):
                                        cmatches[ftype] = True
                                    if 'gt' in cmp and float(
                                            cmp['gt']) < float(val):
                                        cmatches[ftype] = True
                                    if 'lt' in cmp and float(
                                            cmp['lt']) > float(val):
                                        cmatches[ftype] = True
                                except:
                                    logger.warn(
                                        "Not able to apply table conditional formatting to '{}' ({})"
                                        .format(val, cmp))
                # Apply HTML in order of config keys
                bgcol = None
                for cfc in config.table_cond_formatting_colours:
                    for cfck in cfc:  # should always be one, but you never know
                        if cmatches[cfck]:
                            bgcol = cfc[cfck]
                if bgcol is not None:
                    valstring = '<span class="badge" style="background-color:{}">{}</span>'.format(
                        bgcol, valstring)

                # Build HTML
                if not header['scale']:
                    if s_name not in t_rows:
                        t_rows[s_name] = dict()
                    t_rows[s_name][
                        rid] = '<td class="{rid} {h}">{v}</td>'.format(
                            rid=rid, h=hide, v=valstring)

                # coloring with quartiles
                elif header['scale'] == "quart":

                    col_dict = header['col_dict']
                    bar_dict = header.get('bar_dict')
                    col = ' background-color:{};'.format(col_dict.get(s_name))
                    if bar_dict:
                        bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(
                            bar_dict.get(s_name) + 3, col)
                    else:
                        bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(
                            percentage, col)
                    # bar percentage here

                    val_html = '<span class="val">{}</span>'.format(valstring)
                    wrapper_html = '<div class="wrapper">{}{}</div>'.format(
                        bar_html, val_html)
                    if s_name not in t_rows:
                        t_rows[s_name] = dict()
                    t_rows[s_name][
                        rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(
                            rid=rid, h=hide, c=wrapper_html)

                else:
                    if c_scale is not None:
                        col = ' background-color:{};'.format(
                            c_scale.get_colour(val))
                    else:
                        col = ''
                    bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(
                        percentage, col)
                    val_html = '<span class="val">{}</span>'.format(valstring)
                    wrapper_html = '<div class="wrapper">{}{}</div>'.format(
                        bar_html, val_html)

                    if s_name not in t_rows:
                        t_rows[s_name] = dict()
                    t_rows[s_name][
                        rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(
                            rid=rid, h=hide, c=wrapper_html)

                # Is this cell hidden or empty?
                if s_name not in t_rows_empty:
                    t_rows_empty[s_name] = dict()
                t_rows_empty[s_name][rid] = header.get(
                    'hidden', False) or str(val).strip() == ''

        # Remove header if we don't have any filled cells for it
        if sum([len(rows) for rows in t_rows.values()]) == 0:
            t_headers.pop(rid, None)
            t_modal_headers.pop(rid, None)
            logger.debug(
                'Removing header {} from general stats table, as no data'.
                format(k))

    #
    # Put everything together
    #

    # Buttons above the table
    html = ''
    if not config.simple_output:
        # Copy Table Button
        html += """<div class="row">
                    <div class="col-sm-2">
        <button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}">
            <span class="glyphicon glyphicon-copy"></span> Copy table
        </button>
        </div>
        """.format(tid=table_id)

        # <div class="progress-bar progress-bar-warning progress-bar-striped" style="width: 20%">
        #                 <span class="sr-only">20% Complete (warning)</span>
        #               </div>
        #               <div class="progress-bar progress-bar-danger" style="width: 10%">
        #                 <span class="sr-only">10% Complete (danger)</span>
        #               </div>

        if not hide_bar:
            html += """ <div class="col-sm-2">
                <div id="quartiles_bar">
                <div class="progress">
                  <div class="progress-bar progress-bar-q1" style="width: 25%">
                    <span class="sr-only">35% Complete (success)</span> Q1
                  </div>
    
                  <div class="progress-bar progress-bar-q2" style="width: 25%">
                    <span class="sr-only">35% Complete (success)</span> Q2
                  </div>
                
                  <div class="progress-bar progress-bar-q3" style="width: 25%">
                    <span class="sr-only">35% Complete (success)</span> Q3
                  </div>
    
                  <div class="progress-bar progress-bar-q4" style="width: 25%">
                    <span class="sr-only">35% Complete (success)</span> Q4
                  </div>
                </div>
                </div>
                         </div>
                         </div>
                """
        else:
            html += """ </div>
                """
        # Configure Columns Button
        # if len(t_headers) > 1:
        #     html += """
        #     <button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal">
        #         <span class="glyphicon glyphicon-th"></span> Configure Columns
        #     </button>
        #     """.format(tid=table_id)

        # Sort By Highlight button
        # html += """
        # <button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;">
        #     <span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight
        # </button>
        # """.format(tid=table_id)

        # Scatter Plot Button
        # if len(t_headers) > 1:
        #     html += """
        #     <button type="button" class="mqc_table_makeScatter btn btn-default btn-sm" data-toggle="modal" data-target="#tableScatterModal" data-table="#{tid}">
        #         <span class="glyphicon glyphicon glyphicon-stats"></span> Plot
        #     </button>
        #     """.format(tid=table_id)

        # "Showing x of y columns" text
        row_visibilities = [
            all(t_rows_empty[s_name].values()) for s_name in t_rows_empty
        ]
        visible_rows = [x for x in row_visibilities if not x]
        # html += """
        # <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nvisrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small>
        # """.format(tid=table_id, nvisrows=len(visible_rows), nrows=len(t_rows), ncols_vis = (len(t_headers)+1)-hidden_cols, ncols=len(t_headers))

        # Add text
        # html += """
        #         <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nvisrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small>
        #         """.format(tid=table_id, nvisrows=len(visible_rows), nrows=len(t_rows),
        #                    ncols_vis=(len(t_headers) + 1) - hidden_cols, ncols=len(t_headers))

    # Build the table itself
    collapse_class = 'mqc-table-collapse' if len(
        t_rows) > 10 and config.collapse_tables else ''
    html += """
        <div id="{tid}_container" class="mqc_table_container">
            <div class="table-responsive mqc-table-responsive {cc}">
                <table id="{tid}" class="table table-condensed mqc_table" data-title="{title}">
        """.format(tid=table_id, title=table_title, cc=collapse_class)

    # Build the header row
    col1_header = dt.pconfig.get('col1_header', 'Sample Name')
    html += '<thead><tr><th class="rowheader">{}</th>{}</tr></thead>'.format(
        col1_header, ''.join(t_headers.values()))

    # Build the table body
    html += '<tbody>'
    t_row_keys = t_rows.keys()
    if dt.pconfig.get('sortRows') is not False:
        t_row_keys = sorted(t_row_keys)
    for s_name in t_row_keys:
        # Hide the row if all cells are empty or hidden
        row_hidden = ' style="display:none"' if all(
            t_rows_empty[s_name].values()) else ''
        html += '<tr{}>'.format(row_hidden)
        # Sample name row header
        html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format(
            sn=s_name)
        for k in t_headers:
            html += t_rows[s_name].get(k, empty_cells[k])
        html += '</tr>'
    html += '</tbody></table></div>'
    if len(t_rows) > 10 and config.collapse_tables:
        html += '<div class="mqc-table-expand"><span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span></div>'
    html += '</div>'

    # Build the bootstrap modal to customise columns and order
    # if not config.simple_output:
    #     html += """
    # <!-- MultiQC Table Columns Modal -->
    # <div class="modal fade" id="{tid}_configModal" tabindex="-1">
    #   <div class="modal-dialog modal-lg">
    #     <div class="modal-content">
    #       <div class="modal-header">
    #         <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
    #         <h4 class="modal-title">{title}: Columns</h4>
    #       </div>
    #       <div class="modal-body">
    #         <p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p>
    #         <p>
    #             <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button>
    #             <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button>
    #         </p>
    #         <table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table" data-title="{title}">
    #           <thead>
    #             <tr>
    #               <th class="sorthandle" style="text-align:center;">Sort</th>
    #               <th style="text-align:center;">Visible</th>
    #               <th>Group</th>
    #               <th>Column</th>
    #               <th>Description</th>
    #               <th>ID</th>
    #               <th>Scale</th>
    #             </tr>
    #           </thead>
    #           <tbody>
    #             {trows}
    #           </tbody>
    #         </table>
    #     </div>
    #     <div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div>
    # </div> </div> </div>""".format( tid=table_id, title=table_title, trows=''.join(t_modal_headers.values()) )

    # Save the raw values to a file if requested
    if dt.pconfig.get('save_file') is True:
        fn = dt.pconfig.get('raw_data_fn', 'multiqc_{}'.format(table_id))
        util_functions.write_data_file(dt.raw_vals, fn)
        report.saved_raw_data[fn] = dt.raw_vals

    return html
Example #7
0
def matplotlib_bargraph(plotdata, plotsamples, pconfig=None):
    """
    Plot a bargraph with Matplot lib and return a HTML string. Either embeds a base64
    encoded image within HTML or writes the plot and links to it. Should be called by
    plot_bargraph, which properly formats the input data.
    """

    if pconfig is None:
        pconfig = {}

    # Plot group ID
    if pconfig.get("id") is None:
        pconfig["id"] = "mqc_mplplot_" + "".join(random.sample(letters, 10))

    # Sanitise plot ID and check for duplicates
    pconfig["id"] = report.save_htmlid(pconfig["id"])

    # Individual plot IDs
    pids = []
    for k in range(len(plotdata)):
        try:
            name = pconfig["data_labels"][k]
        except:
            name = k + 1
        pid = "mqc_{}_{}".format(pconfig["id"], name)
        pid = report.save_htmlid(pid, skiplint=True)
        pids.append(pid)

    html = (
        '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> '
        + "Flat image plot. Toolbox functions such as highlighting / hiding samples will not work "
        + '(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>'
    )
    html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig["id"])

    # Same defaults as HighCharts for consistency
    default_colors = [
        "#7cb5ec",
        "#434348",
        "#90ed7d",
        "#f7a35c",
        "#8085e9",
        "#f15c80",
        "#e4d354",
        "#2b908f",
        "#f45b5b",
        "#91e8e1",
    ]

    # Counts / Percentages Switch
    if pconfig.get("cpswitch") is not False and not config.simple_output:
        if pconfig.get("cpswitch_c_active", True) is True:
            c_active = "active"
            p_active = ""
        else:
            c_active = ""
            p_active = "active"
            pconfig["stacking"] = "percent"
        c_label = pconfig.get("cpswitch_counts_label", "Counts")
        p_label = pconfig.get("cpswitch_percent_label", "Percentages")
        html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_setcountspcnt"> \n\
            <button class="btn btn-default btn-sm {c_a} counts">{c_l}</button> \n\
            <button class="btn btn-default btn-sm {p_a} pcnt">{p_l}</button> \n\
        </div> '.format(
            c_a=c_active, p_a=p_active, c_l=c_label, p_l=p_label
        )
        if len(plotdata) > 1:
            html += " &nbsp; &nbsp; "

    # Buttons to cycle through different datasets
    if len(plotdata) > 1 and not config.simple_output:
        html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n'
        for k, p in enumerate(plotdata):
            pid = pids[k]
            active = "active" if k == 0 else ""
            try:
                name = pconfig["data_labels"][k]
            except:
                name = k + 1
            html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format(
                a=active, pid=pid, n=name
            )
        html += "</div>\n\n"

    # Go through datasets creating plots
    for pidx, pdata in enumerate(plotdata):

        # Save plot data to file
        fdata = {}
        for d in pdata:
            for didx, dval in enumerate(d["data"]):
                s_name = plotsamples[pidx][didx]
                if s_name not in fdata:
                    fdata[s_name] = dict()
                fdata[s_name][d["name"]] = dval
        if pconfig.get("save_data_file", True):
            util_functions.write_data_file(fdata, pids[pidx])

        # Plot percentage as well as counts
        plot_pcts = [False]
        if pconfig.get("cpswitch") is not False:
            plot_pcts = [False, True]

        # Switch out NaN for 0s so that MatPlotLib doesn't ignore stuff
        for idx, d in enumerate(pdata):
            pdata[idx]["data"] = [x if not math.isnan(x) else 0 for x in d["data"]]

        for plot_pct in plot_pcts:

            # Plot ID
            pid = pids[pidx]
            hide_plot = False
            if plot_pct is True:
                pid = "{}_pc".format(pid)
                if pconfig.get("cpswitch_c_active", True) is True:
                    hide_plot = True
            else:
                if pconfig.get("cpswitch_c_active", True) is not True:
                    hide_plot = True

            # Set up figure

            # Height has a default, then adjusted by the number of samples
            plt_height = len(plotsamples[pidx]) / 2.3  # Default in inches, empirically determined
            plt_height = max(6, plt_height)  # At least 6" tall
            plt_height = min(30, plt_height)  # Cap at 30" tall

            # Use fixed height if pconfig['height'] is set (convert pixels -> inches)
            if "height" in pconfig:
                # Default interactive height in pixels = 512
                # Not perfect replication, but good enough
                plt_height = 6 * (pconfig["height"] / 512)

            bar_width = 0.8

            fig = plt.figure(figsize=(14, plt_height), frameon=False)
            axes = fig.add_subplot(111)
            y_ind = range(len(plotsamples[pidx]))

            # Count totals for each sample
            if plot_pct is True:
                s_totals = [0 for _ in pdata[0]["data"]]
                for series_idx, d in enumerate(pdata):
                    for sample_idx, v in enumerate(d["data"]):
                        s_totals[sample_idx] += v

            # Plot bars
            dlabels = []
            prev_values = None
            for idx, d in enumerate(pdata):
                # Plot percentages
                values = [x for x in d["data"]]
                if len(values) < len(y_ind):
                    values.extend([0] * (len(y_ind) - len(values)))
                if plot_pct is True:
                    for (key, var) in enumerate(values):
                        s_total = s_totals[key]
                        if s_total == 0:
                            values[key] = 0
                        else:
                            values[key] = (float(var + 0.0) / float(s_total)) * 100

                # Get offset for stacked bars
                if idx == 0:
                    prevdata = [0] * len(plotsamples[pidx])
                else:
                    for i, p in enumerate(prevdata):
                        prevdata[i] += prev_values[i]
                # Default colour index
                cidx = idx
                while cidx >= len(default_colors):
                    cidx -= len(default_colors)
                # Save the name of this series
                dlabels.append(d["name"])
                # Add the series of bars to the plot
                axes.barh(
                    y_ind,
                    values,
                    bar_width,
                    left=prevdata,
                    color=d.get("color", default_colors[cidx]),
                    align="center",
                    linewidth=pconfig.get("borderWidth", 0),
                )
                prev_values = values

            # Tidy up axes
            axes.tick_params(
                labelsize=pconfig.get("labelSize", 8), direction="out", left=False, right=False, top=False, bottom=False
            )
            axes.set_xlabel(pconfig.get("ylab", ""))  # I know, I should fix the fact that the config is switched
            axes.set_ylabel(pconfig.get("xlab", ""))
            axes.set_yticks(y_ind)  # Specify where to put the labels
            axes.set_yticklabels(plotsamples[pidx])  # Set y axis sample name labels
            axes.set_ylim((-0.5, len(y_ind) - 0.5))  # Reduce padding around plot area
            if plot_pct is True:
                axes.set_xlim((0, 100))
                # Add percent symbols
                vals = axes.get_xticks()
                axes.set_xticks(axes.get_xticks())
                axes.set_xticklabels(["{:.0f}%".format(x) for x in vals])
            else:
                default_xlimits = axes.get_xlim()
                axes.set_xlim((pconfig.get("ymin", default_xlimits[0]), pconfig.get("ymax", default_xlimits[1])))
            if "title" in pconfig:
                top_gap = 1 + (0.5 / plt_height)
                plt.text(
                    0.5, top_gap, pconfig["title"], horizontalalignment="center", fontsize=16, transform=axes.transAxes
                )
            axes.grid(True, zorder=0, which="both", axis="x", linestyle="-", color="#dedede", linewidth=1)
            axes.set_axisbelow(True)
            axes.spines["right"].set_visible(False)
            axes.spines["top"].set_visible(False)
            axes.spines["bottom"].set_visible(False)
            axes.spines["left"].set_visible(False)
            plt.gca().invert_yaxis()  # y axis is reverse sorted otherwise

            # Hide some labels if we have a lot of samples
            show_nth = max(1, math.ceil(len(pdata[0]["data"]) / 150))
            for idx, label in enumerate(axes.get_yticklabels()):
                if idx % show_nth != 0:
                    label.set_visible(False)

            # Legend
            bottom_gap = -1 * (1 - ((plt_height - 1.5) / plt_height))
            lgd = axes.legend(
                dlabels,
                loc="lower center",
                bbox_to_anchor=(0, bottom_gap, 1, 0.102),
                ncol=5,
                mode="expand",
                fontsize=pconfig.get("labelSize", 8),
                frameon=False,
            )

            # Should this plot be hidden on report load?
            hidediv = ""
            if pidx > 0 or hide_plot:
                hidediv = ' style="display:none;"'

            # Save the plot to the data directory if export is requested
            if config.export_plots:
                for fformat in config.export_plot_formats:
                    # Make the directory if it doesn't already exist
                    plot_dir = os.path.join(config.plots_dir, fformat)
                    if not os.path.exists(plot_dir):
                        os.makedirs(plot_dir)
                    # Save the plot
                    plot_fn = os.path.join(plot_dir, "{}.{}".format(pid, fformat))
                    fig.savefig(plot_fn, format=fformat, bbox_extra_artists=(lgd,), bbox_inches="tight")

            # Output the figure to a base64 encoded string
            if getattr(get_template_mod(), "base64_plots", True) is True:
                img_buffer = io.BytesIO()
                fig.savefig(img_buffer, format="png", bbox_inches="tight")
                b64_img = base64.b64encode(img_buffer.getvalue()).decode("utf8")
                img_buffer.close()
                html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format(
                    pid, hidediv, b64_img
                )

            # Link to the saved image
            else:
                plot_relpath = os.path.join(config.plots_dir_name, "png", "{}.png".format(pid))
                html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(pid, hidediv, plot_relpath)

            plt.close(fig)

    # Close wrapping div
    html += "</div>"

    return html
Example #8
0
def multiqc(analysis_dir, dirs, dirs_depth, no_clean_sname, title,
            report_comment, template, module_tag, module, exclude, outdir,
            ignore, ignore_samples, sample_names, file_list, filename,
            make_data_dir, no_data_dir, data_format, zip_data_dir, force,
            ignore_symlinks, export_plots, plots_flat, plots_interactive, lint,
            make_pdf, no_megaqc_upload, config_file, cl_config, verbose, quiet,
            **kwargs):
    """MultiQC aggregates results from bioinformatics analyses across many samples into a single report.

        It searches a given directory for analysis logs and compiles a HTML report.
        It's a general use tool, perfect for summarising the output from numerous
        bioinformatics tools.

        To run, supply with one or more directory to scan for analysis results.
        To run here, use 'multiqc .'

        See http://multiqc.info for more details.

        Author: Phil Ewels (http://phil.ewels.co.uk)
    """

    # Set up logging level
    loglevel = log.LEVELS.get(min(verbose, 1), "INFO")
    if quiet:
        loglevel = 'WARNING'
    log.init_log(logger, loglevel=loglevel)

    # Load config files
    plugin_hooks.mqc_trigger('before_config')
    config.mqc_load_userconfig(config_file)
    plugin_hooks.mqc_trigger('config_loaded')

    # Command-line config YAML
    if len(cl_config) > 0:
        config.mqc_cl_config(cl_config)

    # Log the command used to launch MultiQC
    report.multiqc_command = " ".join(sys.argv)
    logger.debug("Command used: {}".format(report.multiqc_command))

    # Check that we're running the latest version of MultiQC
    if config.no_version_check is not True:
        try:
            response = urlopen('http://multiqc.info/version.php?v={}'.format(
                config.short_version),
                               timeout=5)
            remote_version = response.read().decode('utf-8').strip()
            if version.StrictVersion(re.sub(
                    '[^0-9\.]', '', remote_version)) > version.StrictVersion(
                        re.sub('[^0-9\.]', '', config.short_version)):
                logger.warn(
                    'MultiQC Version {} now available!'.format(remote_version))
            else:
                logger.debug(
                    'Latest MultiQC version is {}'.format(remote_version))
        except Exception as e:
            logger.debug(
                'Could not connect to multiqc.info for version check: {}'.
                format(e))

    # Set up key variables (overwrite config vars from command line)
    if template is not None:
        config.template = template
    if title is not None:
        config.title = title
    if report_comment is not None:
        config.report_comment = report_comment
    if dirs is True:
        config.prepend_dirs = dirs
    if dirs_depth is not None:
        config.prepend_dirs = True
        config.prepend_dirs_depth = dirs_depth
    config.analysis_dir = analysis_dir
    if outdir is not None:
        config.output_dir = outdir
    if no_clean_sname:
        config.fn_clean_sample_names = False
        logger.info("Not cleaning sample names")
    if make_data_dir:
        config.make_data_dir = True
    if no_data_dir:
        config.make_data_dir = False
    if force:
        config.force = True
    if ignore_symlinks:
        config.ignore_symlinks = True
    if zip_data_dir:
        config.zip_data_dir = True
    if data_format is not None:
        config.data_format = data_format
    if export_plots:
        config.export_plots = True
    if plots_flat:
        config.plots_force_flat = True
    if plots_interactive:
        config.plots_force_interactive = True
    if lint:
        config.lint = True
        lint_helpers.run_tests()
    if make_pdf:
        config.template = 'simple'
    if no_megaqc_upload:
        config.megaqc_upload = False
    else:
        config.megaqc_upload = True
    if sample_names:
        config.load_sample_names(sample_names)
    if module_tag is not None:
        config.module_tag = module_tag
    config.kwargs = kwargs  # Plugin command line options

    plugin_hooks.mqc_trigger('execution_start')

    logger.info("This is MultiQC v{}".format(__version__))
    logger.debug("Command     : {}".format(' '.join(sys.argv)))
    logger.debug("Working dir : {}".format(os.getcwd()))
    if make_pdf:
        logger.info('--pdf specified. Using non-interactive HTML template.')
    logger.info("Template    : {}".format(config.template))
    if lint:
        logger.info('--lint specified. Being strict with validation.')

    # Add files if --file-list option is given
    if file_list:
        if len(analysis_dir) > 1:
            raise ValueError(
                "If --file-list is giving, analysis_dir should have only one plain text file."
            )
        config.analysis_dir = []
        with (open(analysis_dir[0])) as in_handle:
            for line in in_handle:
                if os.path.exists(line.strip()):
                    path = os.path.abspath(line.strip())
                    config.analysis_dir.append(path)
        if len(config.analysis_dir) == 0:
            logger.error(
                "No files or directories were added from {} using --file-list option."
                .format(analysis_dir[0]))
            logger.error(
                "Please, check that {} contains correct paths.".format(
                    analysis_dir[0]))
            raise ValueError("Any files or directories to be searched.")

    if len(ignore) > 0:
        logger.debug(
            "Ignoring files, directories and paths that match: {}".format(
                ", ".join(ignore)))
        config.fn_ignore_files.extend(ignore)
        config.fn_ignore_dirs.extend(ignore)
        config.fn_ignore_paths.extend(ignore)
    if len(ignore_samples) > 0:
        logger.debug("Ignoring sample names that match: {}".format(
            ", ".join(ignore_samples)))
        config.sample_names_ignore.extend(ignore_samples)
    if filename == 'stdout':
        config.output_fn = sys.stdout
        logger.info("Printing report to stdout")
    else:
        if title is not None and filename is None:
            filename = re.sub('[^\w\.-]', '', re.sub('[-\s]+', '-',
                                                     title)).strip()
            filename += '_multiqc_report'
        if filename is not None:
            if filename.endswith('.html'):
                filename = filename[:-5]
            config.output_fn_name = filename
            config.data_dir_name = '{}_data'.format(filename)
        if not config.output_fn_name.endswith('.html'):
            config.output_fn_name = '{}.html'.format(config.output_fn_name)

    # Print some status updates
    if config.title is not None:
        logger.info("Report title: {}".format(config.title))
    if dirs:
        logger.info("Prepending directory to sample names")
    for d in config.analysis_dir:
        logger.info("Searching '{}'".format(d))

    # Prep module configs
    config.top_modules = [
        m if type(m) is dict else {
            m: {}
        } for m in config.top_modules
    ]
    config.module_order = [
        m if type(m) is dict else {
            m: {}
        } for m in config.module_order
    ]
    mod_keys = [list(m.keys())[0] for m in config.module_order]

    # Lint the module configs
    if config.lint:
        for m in config.avail_modules.keys():
            if m not in mod_keys:
                errmsg = "LINT: Module '{}' not found in config.module_order".format(
                    m)
                logger.error(errmsg)
                report.lint_errors.append(errmsg)
            else:
                for mo in config.module_order:
                    if m != 'custom_content' and m in mo.keys(
                    ) and 'module_tag' not in mo[m]:
                        errmsg = "LINT: Module '{}' in config.module_order did not have 'module_tag' config".format(
                            m)
                        logger.error(errmsg)
                        report.lint_errors.append(errmsg)

    # Get the avaiable tags to decide which modules to run.
    modules_from_tags = set()
    if config.module_tag is not None:
        tags = config.module_tag
        for m in config.module_order:
            module_name = list(m.keys())[0]  # only one name in each dict
            for tag in tags:
                for t in m[module_name].get('module_tag', []):
                    if tag.lower() == t.lower():
                        modules_from_tags.add(module_name)

    # Get the list of modules we want to run, in the order that we want them
    run_modules = [
        m for m in config.top_modules
        if list(m.keys())[0] in config.avail_modules.keys()
    ]
    run_modules.extend([{
        m: {}
    } for m in config.avail_modules.keys()
                        if m not in mod_keys and m not in run_modules])
    run_modules.extend([
        m for m in config.module_order
        if list(m.keys())[0] in config.avail_modules.keys() and list(m.keys())
        [0] not in [list(rm.keys())[0] for rm in run_modules]
    ])

    if module:
        run_modules = [m for m in run_modules if list(m.keys())[0] in module]
        logger.info('Only using modules {}'.format(', '.join(module)))
    elif modules_from_tags:
        run_modules = [
            m for m in run_modules if list(m.keys())[0] in modules_from_tags
        ]
        logger.info("Only using modules with '{}' tag".format(
            ', '.join(module_tag)))
    if exclude:
        logger.info("Excluding modules '{}'".format("', '".join(exclude)))
        if 'general_stats' in exclude:
            config.skip_generalstats = True
            exclude = tuple(x for x in exclude if x != 'general_stats')
        run_modules = [
            m for m in run_modules if list(m.keys())[0] not in exclude
        ]
    if len(run_modules) == 0:
        logger.critical('No analysis modules specified!')
        sys.exit(1)
    run_module_names = [list(m.keys())[0] for m in run_modules]
    logger.debug("Analysing modules: {}".format(', '.join(run_module_names)))

    # Create the temporary working directories
    tmp_dir = tempfile.mkdtemp()
    logger.debug(
        'Using temporary directory for creating report: {}'.format(tmp_dir))
    config.data_tmp_dir = os.path.join(tmp_dir, 'multiqc_data')
    if filename != 'stdout' and config.make_data_dir == True:
        config.data_dir = config.data_tmp_dir
        os.makedirs(config.data_dir)
    else:
        config.data_dir = None
    config.plots_tmp_dir = os.path.join(tmp_dir, 'multiqc_plots')
    if filename != 'stdout' and config.export_plots == True:
        config.plots_dir = config.plots_tmp_dir
        os.makedirs(config.plots_dir)

    # Load the template
    template_mod = config.avail_templates[config.template].load()

    # Add an output subdirectory if specified by template
    try:
        config.output_dir = os.path.join(config.output_dir,
                                         template_mod.output_subdir)
    except AttributeError:
        pass  # No subdirectory variable given

    # Add custom content section names
    try:
        if 'custom_content' in run_module_names:
            run_module_names.extend(config.custom_data.keys())
    except AttributeError:
        pass  # custom_data not in config

    # Get the list of files to search
    report.get_filelist(run_module_names)

    # Run the modules!
    plugin_hooks.mqc_trigger('before_modules')
    report.modules_output = list()
    sys_exit_code = 0
    for mod_dict in run_modules:
        try:
            this_module = list(mod_dict.keys())[0]
            mod_cust_config = list(mod_dict.values())[0]
            mod = config.avail_modules[this_module].load()
            mod.mod_cust_config = mod_cust_config  # feels bad doing this, but seems to work
            output = mod()
            if type(output) != list:
                output = [output]
            for m in output:
                report.modules_output.append(m)

            # Copy over css & js files if requested by the theme
            try:
                for to, path in report.modules_output[-1].css.items():
                    copy_to = os.path.join(tmp_dir, to)
                    os.makedirs(os.path.dirname(copy_to))
                    shutil.copyfile(path, copy_to)
            except OSError as e:
                if e.errno == errno.EEXIST:
                    pass
                else:
                    raise
            except AttributeError:
                pass
            try:
                for to, path in report.modules_output[-1].js.items():
                    copy_to = os.path.join(tmp_dir, to)
                    os.makedirs(os.path.dirname(copy_to))
                    shutil.copyfile(path, copy_to)
            except OSError as e:
                if e.errno == errno.EEXIST:
                    pass
                else:
                    raise
            except AttributeError:
                pass

        except UserWarning:
            logger.debug("No samples found: {}".format(
                list(mod_dict.keys())[0]))
        except KeyboardInterrupt:
            shutil.rmtree(tmp_dir)
            logger.critical("User Cancelled Execution!\n{eq}\n{tb}{eq}\n".
                            format(eq=('=' * 60), tb=traceback.format_exc()) +
                            "User Cancelled Execution!\nExiting MultiQC...")
            sys.exit(1)
        except:
            # Flag the error, but carry on
            logger.error("Oops! The '{}' MultiQC module broke... \n".format(this_module) + \
                      "  Please copy the following traceback and report it at " + \
                      "https://github.com/ewels/MultiQC/issues \n" + \
                      "  If possible, please include a log file that triggers the error - " + \
                      "the last file found was:\n" + \
                      "    {}\n".format(report.last_found_file) + \
                      ('='*60)+"\nModule {} raised an exception: {}".format(
                          this_module, traceback.format_exc()) + ('='*60))
            sys_exit_code = 1

    # Did we find anything?
    if len(report.modules_output) == 0:
        logger.warn("No analysis results found. Cleaning up..")
        shutil.rmtree(tmp_dir)
        logger.info("MultiQC complete")
        # Exit with an error code if a module broke
        sys.exit(sys_exit_code)

    # Sort the report sections if we have a config
    if len(getattr(config, 'report_section_order', {})) > 0:
        section_id_order = {}
        idx = 10
        for mod in reversed(report.modules_output):
            section_id_order[mod.anchor] = idx
            idx += 10
        for anchor, ss in config.report_section_order.items():
            if anchor not in section_id_order.keys():
                continue
            if ss.get('order') is not None:
                section_id_order[anchor] = ss['order']
            if ss.get('after') in section_id_order.keys():
                section_id_order[anchor] = section_id_order[ss['after']] + 1
            if ss.get('before') in section_id_order.keys():
                section_id_order[anchor] = section_id_order[ss['before']] - 1
        sorted_ids = sorted(section_id_order, key=section_id_order.get)
        report.modules_output = [
            mod for i in reversed(sorted_ids) for mod in report.modules_output
            if mod.anchor == i
        ]

    plugin_hooks.mqc_trigger('after_modules')

    # Remove empty data sections from the General Stats table
    empty_keys = [
        i for i, d in enumerate(report.general_stats_data[:]) if len(d) == 0
    ]
    empty_keys.sort(reverse=True)
    for i in empty_keys:
        del report.general_stats_data[i]
        del report.general_stats_headers[i]
    # Add general-stats IDs to table row headers
    for idx, h in enumerate(report.general_stats_headers):
        for k in h.keys():
            if 'rid' not in h[k]:
                h[k]['rid'] = re.sub(r'\W+', '_', k).strip().strip('_')
            ns_html = re.sub(r'\W+', '_',
                             h[k]['namespace']).strip().strip('_').lower()
            report.general_stats_headers[idx][k]['rid'] = report.save_htmlid(
                'mqc-generalstats-{}-{}'.format(ns_html, h[k]['rid']))
    # Generate the General Statistics HTML & write to file
    if len(report.general_stats_data) > 0:
        pconfig = {
            'id': 'general_stats_table',
            'table_title': 'General Statistics',
            'save_file': True,
            'raw_data_fn': 'multiqc_general_stats'
        }
        report.general_stats_html = table.plot(report.general_stats_data,
                                               report.general_stats_headers,
                                               pconfig)
    else:
        config.skip_generalstats = True

    # Write the report sources to disk
    if config.data_dir is not None:
        report.data_sources_tofile()
    # Compress the report plot JSON data
    logger.info("Compressing plot data")
    report.plot_compressed_json = report.compress_json(report.plot_data)

    plugin_hooks.mqc_trigger('before_report_generation')

    # Data Export / MegaQC integration - save report data to file or send report data to an API endpoint
    if (config.data_dump_file or config.megaqc_url) and config.megaqc_upload:
        multiqc_json_dump = megaqc.multiqc_dump_json(report)
        if config.data_dump_file:
            util_functions.write_data_file(multiqc_json_dump, 'multiqc_data',
                                           False, 'json')
        if config.megaqc_url:
            megaqc.multiqc_api_post(multiqc_json_dump)

    # Make the final report path & data directories
    if filename != 'stdout':
        config.output_fn = os.path.join(config.output_dir,
                                        config.output_fn_name)
        config.data_dir = os.path.join(config.output_dir, config.data_dir_name)
        # Check for existing reports and remove if -f was specified
        if os.path.exists(
                config.output_fn) or (config.make_data_dir
                                      and os.path.exists(config.data_dir)):
            if config.force:
                if os.path.exists(config.output_fn):
                    logger.warning(
                        "Deleting    : {}   (-f was specified)".format(
                            os.path.relpath(config.output_fn)))
                    os.remove(config.output_fn)
                if config.make_data_dir and os.path.exists(config.data_dir):
                    logger.warning(
                        "Deleting    : {}   (-f was specified)".format(
                            os.path.relpath(config.data_dir)))
                    shutil.rmtree(config.data_dir)
            else:
                # Set up the base names of the report and the data dir
                report_num = 1
                report_base, report_ext = os.path.splitext(
                    config.output_fn_name)
                dir_base = os.path.basename(config.data_dir)

                # Iterate through appended numbers until we find one that's free
                while os.path.exists(
                        config.output_fn) or (config.make_data_dir and
                                              os.path.exists(config.data_dir)):
                    config.output_fn = os.path.join(
                        config.output_dir,
                        "{}_{}{}".format(report_base, report_num, report_ext))
                    config.data_dir = os.path.join(
                        config.output_dir,
                        "{}_{}".format(dir_base, report_num))
                    report_num += 1

                config.output_fn_name = os.path.basename(config.output_fn)
                config.data_dir_name = os.path.basename(config.data_dir)
                logger.warning(
                    "Previous MultiQC output found! Adjusting filenames..")
                logger.warning(
                    "Use -f or --force to overwrite existing reports instead")

        # Make directories for report if needed
        if not os.path.exists(os.path.dirname(config.output_fn)):
            os.makedirs(os.path.dirname(config.output_fn))
        logger.info("Report      : {}".format(os.path.relpath(
            config.output_fn)))

        if config.make_data_dir == False:
            logger.info("Data        : None")
        else:
            # Make directories for data_dir
            logger.info("Data        : {}".format(
                os.path.relpath(config.data_dir)))
            if not os.path.exists(config.data_dir):
                os.makedirs(config.data_dir)
            # Modules have run, so data directory should be complete by now. Move its contents.
            for f in os.listdir(config.data_tmp_dir):
                fn = os.path.join(config.data_tmp_dir, f)
                logger.debug("Moving data file from '{}' to '{}'".format(
                    fn, config.data_dir))
                shutil.move(fn, config.data_dir)

        # Copy across the static plot images if requested
        if config.export_plots:
            config.plots_dir = os.path.join(config.output_dir,
                                            config.plots_dir_name)
            if os.path.exists(config.plots_dir):
                if config.force:
                    logger.warning(
                        "Deleting    : {}   (-f was specified)".format(
                            os.path.relpath(config.plots_dir)))
                    shutil.rmtree(config.plots_dir)
                else:
                    logger.error("Output directory {} already exists.".format(
                        config.plots_dir))
                    logger.info(
                        "Use -f or --force to overwrite existing reports")
                    shutil.rmtree(tmp_dir)
                    sys.exit(1)
            os.makedirs(config.plots_dir)
            logger.info("Plots       : {}".format(
                os.path.relpath(config.plots_dir)))

            # Modules have run, so plots directory should be complete by now. Move its contents.
            for f in os.listdir(config.plots_tmp_dir):
                fn = os.path.join(config.plots_tmp_dir, f)
                logger.debug("Moving plots directory from '{}' to '{}'".format(
                    fn, config.plots_dir))
                shutil.move(fn, config.plots_dir)

    plugin_hooks.mqc_trigger('before_template')

    # Load in parent template files first if a child theme
    try:
        parent_template = config.avail_templates[
            template_mod.template_parent].load()
        copy_tree(parent_template.template_dir, tmp_dir)
    except AttributeError:
        pass  # Not a child theme

    # Copy the template files to the tmp directory (distutils overwrites parent theme files)
    copy_tree(template_mod.template_dir, tmp_dir)

    # Function to include file contents in Jinja template
    def include_file(name, fdir=tmp_dir, b64=False):
        try:
            if fdir is None:
                fdir = ''
            if b64:
                with io.open(os.path.join(fdir, name), "rb") as f:
                    return base64.b64encode(f.read()).decode('utf-8')
            else:
                with io.open(os.path.join(fdir, name), "r",
                             encoding='utf-8') as f:
                    return f.read()
        except (OSError, IOError) as e:
            logger.error("Could not include file '{}': {}".format(name, e))

    # Load the report template
    try:
        env = jinja2.Environment(loader=jinja2.FileSystemLoader(tmp_dir))
        env.globals['include_file'] = include_file
        j_template = env.get_template(template_mod.base_fn)
    except:
        raise IOError("Could not load {} template file '{}'".format(
            config.template, template_mod.base_fn))

    # Use jinja2 to render the template and overwrite
    config.analysis_dir = [os.path.realpath(d) for d in config.analysis_dir]
    report_output = j_template.render(report=report, config=config)
    if filename == 'stdout':
        print(report_output.encode('utf-8'), file=sys.stdout)
    else:
        try:
            with io.open(config.output_fn, "w", encoding='utf-8') as f:
                print(report_output, file=f)
        except IOError as e:
            raise IOError("Could not print report to '{}' - {}".format(
                config.output_fn, IOError(e)))

        # Copy over files if requested by the theme
        try:
            for f in template_mod.copy_files:
                fn = os.path.join(tmp_dir, f)
                dest_dir = os.path.join(os.path.dirname(config.output_fn), f)
                copy_tree(fn, dest_dir)
        except AttributeError:
            pass  # No files to copy

    # Clean up temporary directory
    shutil.rmtree(tmp_dir)

    # Zip the data directory if requested
    if config.zip_data_dir and config.data_dir is not None:
        shutil.make_archive(config.data_dir, 'zip', config.data_dir)
        shutil.rmtree(config.data_dir)

    # Try to create a PDF if requested
    if make_pdf:
        try:
            pdf_fn_name = config.output_fn.replace('.html', '.pdf')
            pandoc_call = [
                'pandoc', '--standalone', config.output_fn, '--output',
                pdf_fn_name, '--pdf-engine=xelatex', '-V',
                'documentclass=article', '-V', 'geometry=margin=1in', '-V',
                'title='
            ]
            if config.pandoc_template is not None:
                pandoc_call.append('--template={}'.format(
                    config.pandoc_template))
            logger.debug(
                "Attempting Pandoc conversion to PDF with following command:\n{}"
                .format(' '.join(pandoc_call)))
            pdf_exit_code = subprocess.call(pandoc_call)
            if pdf_exit_code != 0:
                logger.error(
                    "Error creating PDF! Pandoc returned a non-zero exit code."
                )
            else:
                logger.info("PDF Report  : {}".format(pdf_fn_name))
        except OSError as e:
            if e.errno == os.errno.ENOENT:
                logger.error(
                    'Error creating PDF - pandoc not found. Is it installed? http://pandoc.org/'
                )
            else:
                logger.error(
                    "Error creating PDF! Something went wrong when creating the PDF\n"
                    + ('=' * 60) + "\n{}\n".format(traceback.format_exc()) +
                    ('=' * 60))

    plugin_hooks.mqc_trigger('execution_finish')

    logger.info("MultiQC complete")

    if lint and len(report.lint_errors) > 0:
        logger.error("Found {} linting errors!\n{}".format(
            len(report.lint_errors), "\n".join(report.lint_errors)))
        sys_exit_code = 1

    # Move the log file into the data directory
    log.move_tmp_log(logger)

    # Exit with an error code if a module broke
    sys.exit(sys_exit_code)
Example #9
0
def matplotlib_bargraph(plotdata, plotsamples, pconfig=None):
    """
    Plot a bargraph with Matplot lib and return a HTML string. Either embeds a base64
    encoded image within HTML or writes the plot and links to it. Should be called by
    plot_bargraph, which properly formats the input data.
    """

    if pconfig is None:
        pconfig = {}

    # Plot group ID
    if pconfig.get('id') is None:
        pconfig['id'] = 'mqc_mplplot_' + ''.join(random.sample(letters, 10))

    # Sanitise plot ID and check for duplicates
    pconfig['id'] = report.save_htmlid(pconfig['id'])

    # Individual plot IDs
    pids = []
    for k in range(len(plotdata)):
        try:
            name = pconfig['data_labels'][k]
        except:
            name = k + 1
        pid = 'mqc_{}_{}'.format(pconfig['id'], name)
        pid = report.save_htmlid(pid, skiplint=True)
        pids.append(pid)

    html = '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> ' + \
          'Flat image plot. Toolbox functions such as highlighting / hiding samples will not work ' + \
          '(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>'
    html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig['id'])

    # Same defaults as HighCharts for consistency
    default_colors = [
        '#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80',
        '#e4d354', '#2b908f', '#f45b5b', '#91e8e1'
    ]

    # Counts / Percentages Switch
    if pconfig.get('cpswitch') is not False and not config.simple_output:
        if pconfig.get('cpswitch_c_active', True) is True:
            c_active = 'active'
            p_active = ''
        else:
            c_active = ''
            p_active = 'active'
            pconfig['stacking'] = 'percent'
        c_label = pconfig.get('cpswitch_counts_label', 'Counts')
        p_label = pconfig.get('cpswitch_percent_label', 'Percentages')
        html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_setcountspcnt"> \n\
            <button class="btn btn-default btn-sm {c_a} counts">{c_l}</button> \n\
            <button class="btn btn-default btn-sm {p_a} pcnt">{p_l}</button> \n\
        </div> '.format(c_a=c_active, p_a=p_active, c_l=c_label, p_l=p_label)
        if len(plotdata) > 1:
            html += ' &nbsp; &nbsp; '

    # Buttons to cycle through different datasets
    if len(plotdata) > 1 and not config.simple_output:
        html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n'
        for k, p in enumerate(plotdata):
            pid = pids[k]
            active = 'active' if k == 0 else ''
            try:
                name = pconfig['data_labels'][k]
            except:
                name = k + 1
            html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format(
                a=active, pid=pid, n=name)
        html += '</div>\n\n'

    # Go through datasets creating plots
    for pidx, pdata in enumerate(plotdata):

        # Save plot data to file
        fdata = {}
        for d in pdata:
            for didx, dval in enumerate(d['data']):
                s_name = plotsamples[pidx][didx]
                if s_name not in fdata:
                    fdata[s_name] = dict()
                fdata[s_name][d['name']] = dval
        util_functions.write_data_file(fdata, pids[pidx])

        # Plot percentage as well as counts
        plot_pcts = [False]
        if pconfig.get('cpswitch') is not False:
            plot_pcts = [False, True]

        # Switch out NaN for 0s so that MatPlotLib doesn't ignore stuff
        for idx, d in enumerate(pdata):
            pdata[idx]['data'] = [
                x if not math.isnan(x) else 0 for x in d['data']
            ]

        for plot_pct in plot_pcts:

            # Plot ID
            pid = pids[pidx]
            hide_plot = False
            if plot_pct is True:
                pid = '{}_pc'.format(pid)
                if pconfig.get('cpswitch_c_active', True) is True:
                    hide_plot = True
            else:
                if pconfig.get('cpswitch_c_active', True) is not True:
                    hide_plot = True

            # Set up figure
            plt_height = len(plotsamples[pidx]) / 2.3
            plt_height = max(6, plt_height)  # At least 6" tall
            plt_height = min(30, plt_height)  # Cap at 30" tall
            bar_width = 0.8

            fig = plt.figure(figsize=(14, plt_height), frameon=False)
            axes = fig.add_subplot(111)
            y_ind = range(len(plotsamples[pidx]))

            # Count totals for each sample
            if plot_pct is True:
                s_totals = [0 for _ in pdata[0]['data']]
                for series_idx, d in enumerate(pdata):
                    for sample_idx, v in enumerate(d['data']):
                        s_totals[sample_idx] += v

            # Plot bars
            dlabels = []
            for idx, d in enumerate(pdata):
                # Plot percentages
                values = d['data']
                if len(values) < len(y_ind):
                    values.extend([0] * (len(y_ind) - len(values)))
                if plot_pct is True:
                    for (key, var) in enumerate(values):
                        s_total = s_totals[key]
                        if s_total == 0:
                            values[key] = 0
                        else:
                            values[key] = (float(var + 0.0) /
                                           float(s_total)) * 100

                # Get offset for stacked bars
                if idx == 0:
                    prevdata = [0] * len(plotsamples[pidx])
                else:
                    for i, p in enumerate(prevdata):
                        prevdata[i] += pdata[idx - 1]['data'][i]
                # Default colour index
                cidx = idx
                while cidx >= len(default_colors):
                    cidx -= len(default_colors)
                # Save the name of this series
                dlabels.append(d['name'])
                # Add the series of bars to the plot
                axes.barh(y_ind,
                          values,
                          bar_width,
                          left=prevdata,
                          color=d.get('color', default_colors[cidx]),
                          align='center',
                          linewidth=pconfig.get('borderWidth', 0))

            # Tidy up axes
            axes.tick_params(labelsize=8,
                             direction='out',
                             left=False,
                             right=False,
                             top=False,
                             bottom=False)
            axes.set_xlabel(
                pconfig.get('ylab', '')
            )  # I know, I should fix the fact that the config is switched
            axes.set_ylabel(pconfig.get('xlab', ''))
            axes.set_yticks(y_ind)  # Specify where to put the labels
            axes.set_yticklabels(
                plotsamples[pidx])  # Set y axis sample name labels
            axes.set_ylim(
                (-0.5, len(y_ind) - 0.5))  # Reduce padding around plot area
            if plot_pct is True:
                axes.set_xlim((0, 100))
                # Add percent symbols
                vals = axes.get_xticks()
                axes.set_xticklabels(['{:.0f}%'.format(x) for x in vals])
            else:
                default_xlimits = axes.get_xlim()
                axes.set_xlim((pconfig.get('ymin', default_xlimits[0]),
                               pconfig.get('ymax', default_xlimits[1])))
            if 'title' in pconfig:
                top_gap = 1 + (0.5 / plt_height)
                plt.text(0.5,
                         top_gap,
                         pconfig['title'],
                         horizontalalignment='center',
                         fontsize=16,
                         transform=axes.transAxes)
            axes.grid(True,
                      zorder=0,
                      which='both',
                      axis='x',
                      linestyle='-',
                      color='#dedede',
                      linewidth=1)
            axes.set_axisbelow(True)
            axes.spines['right'].set_visible(False)
            axes.spines['top'].set_visible(False)
            axes.spines['bottom'].set_visible(False)
            axes.spines['left'].set_visible(False)
            plt.gca().invert_yaxis()  # y axis is reverse sorted otherwise

            # Hide some labels if we have a lot of samples
            show_nth = max(1, math.ceil(len(pdata[0]['data']) / 150))
            for idx, label in enumerate(axes.get_yticklabels()):
                if idx % show_nth != 0:
                    label.set_visible(False)

            # Legend
            bottom_gap = -1 * (1 - ((plt_height - 1.5) / plt_height))
            lgd = axes.legend(dlabels,
                              loc='lower center',
                              bbox_to_anchor=(0, bottom_gap, 1, .102),
                              ncol=5,
                              mode='expand',
                              fontsize=8,
                              frameon=False)

            # Should this plot be hidden on report load?
            hidediv = ''
            if pidx > 0 or hide_plot:
                hidediv = ' style="display:none;"'

            # Save the plot to the data directory if export is requested
            if config.export_plots:
                for fformat in config.export_plot_formats:
                    # Make the directory if it doesn't already exist
                    plot_dir = os.path.join(config.plots_dir, fformat)
                    if not os.path.exists(plot_dir):
                        os.makedirs(plot_dir)
                    # Save the plot
                    plot_fn = os.path.join(plot_dir,
                                           '{}.{}'.format(pid, fformat))
                    fig.savefig(plot_fn,
                                format=fformat,
                                bbox_extra_artists=(lgd, ),
                                bbox_inches='tight')

            # Output the figure to a base64 encoded string
            if getattr(get_template_mod(), 'base64_plots', True) is True:
                img_buffer = io.BytesIO()
                fig.savefig(img_buffer, format='png', bbox_inches='tight')
                b64_img = base64.b64encode(
                    img_buffer.getvalue()).decode('utf8')
                img_buffer.close()
                html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format(
                    pid, hidediv, b64_img)

            # Link to the saved image
            else:
                plot_relpath = os.path.join(config.plots_dir_name, 'png',
                                            '{}.png'.format(pid))
                html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(
                    pid, hidediv, plot_relpath)

            plt.close(fig)

    # Close wrapping div
    html += '</div>'

    report.num_mpl_plots += 1

    return html
Example #10
0
    def general_stats_sample_meta(self):
        """ Add metadata about each sample to the General Stats table """

        meta = report.ngi.get('sample_meta')
        if meta is not None and len(meta) > 0:

            log.info('Found {} samples in StatusDB'.format(len(meta)))

            # Write to file
            util_functions.write_data_file(meta, 'ngi_meta')

            # Add to General Stats table
            gsdata = dict()
            formats = dict()
            s_names = dict()
            ngi_ids = dict()
            conc_units = ''
            for sid in meta:

                # Find first sample name matching this sample ID
                s_name = None
                for x in sorted(self.s_names):
                    if sid in x:
                        s_name = x
                        s_names[s_name] = x
                        ngi_ids[s_name] = sid
                        break

                # Skip this sample if we don't have any matching data
                if s_name is None:
                    log.debug("Skipping StatusDB metadata for sample {} as no bioinfo report logs found.".format(sid))
                    continue

                # Make a dict to hold new data for General Stats
                gsdata[s_name] = dict()

                # NGI name
                try:
                    gsdata[s_name]['user_sample_name'] = report.ngi['ngi_names'][ngi_ids[s_name]]
                except KeyError:
                    pass

                # RIN score
                try:
                    gsdata[s_name]['initial_qc_rin'] = meta[sid]['initial_qc']['rin']
                except KeyError:
                    pass

                # Try to figure out which library prep was used
                seq_lp = None
                for lp in sorted(meta[sid].get('library_prep', {}).keys()):
                    try:
                        if len(meta[sid]['library_prep'][lp]['sample_run_metrics']) > 0:
                            if seq_lp is None:
                                seq_lp = lp
                            else:
                                seq_lp = None
                                log.warn('Found multiple sequenced lib preps for {} - skipping metadata'.format(sid))
                                break
                    except KeyError:
                        pass
                if seq_lp is not None:
                    try:
                        if meta[sid]['library_prep'][lp]['amount_taken_(ng)'] is not None:
                            gsdata[s_name]['amount_taken'] = meta[sid]['library_prep'][lp]['amount_taken_(ng)']
                    except KeyError:
                        pass
                    try:
                        for lv in sorted(meta[sid]['library_prep'][lp]['library_validation'].keys()):
                            gsdata[s_name]['lp_concentration'] = meta[sid]['library_prep'][lp]['library_validation'][lv]['concentration']
                            formats[s_name] = meta[sid]['library_prep'][lp]['library_validation'][lv]['conc_units']
                    except KeyError:
                        pass

            log.info("Matched meta for {} samples from StatusDB with report sample names".format(len(s_names)))
            if len(s_names) == 0:
                return None

            # Deal with having more than one initial QC concentration unit
            formats_set = set(formats.values())
            if len(formats_set) > 1:
                log.warning("Mixture of library_validation concentration units! Found: {}".format(", ".join(formats_set)))
                for s_name in gsdata:
                    try:
                        gsdata[s_name]['lp_concentration'] = '{} {}'.format(gsdata[s_name]['lp_concentration'], formats[s_name])
                    except KeyError:
                        pass
            elif len(formats_set) == 1:
                conc_units = formats_set.pop()

            # Decide on whether to show or hide conc & amount taken based on range
            conc_hidden = True
            amounts_hidden = True
            try:
                concs = [gsdata[x]['lp_concentration'] for x in gsdata]
                if max(concs) - min(concs) > 50:
                    conc_hidden = False
            except (KeyError, ValueError):
                conc_hidden = False
            try:
                amounts = [gsdata[x]['amount_taken'] for x in gsdata]
                if max(amounts) - min(amounts) > 10:
                    amounts_hidden = False
            except KeyError:
                amounts_hidden = False

            # Prepend columns to the General Stats table (far left)
            gsheaders_prepend = OrderedDict()
            gsheaders_prepend['user_sample_name'] = {
                'namespace': 'NGI',
                'title': 'Name',
                'description': 'User sample ID',
                'scale': False
            }
            report.general_stats_data.insert(0, gsdata)
            report.general_stats_headers.insert(0, gsheaders_prepend)

            # Add columns to the far right of the General Stats table
            gsheaders = OrderedDict()
            gsheaders['initial_qc_rin'] = {
                'namespace': 'NGI',
                'title': 'RIN',
                'description': 'Initial QC: RNA Integrity Number',
                'min': 0,
                'max': 10,
                'scale': 'YlGn',
                'format': '{:,.2f}'
            }
            gsheaders['lp_concentration'] = {
                'namespace': 'NGI',
                'title': 'Lib Conc. ({})'.format(conc_units),
                'description': 'Library Prep: Concentration ({})'.format(conc_units),
                'min': 0,
                'scale': 'YlGn',
                'format': '{:.,0f}',
                'hidden': conc_hidden
            }
            gsheaders['amount_taken'] = {
                'namespace': 'NGI',
                'title': 'Amount Taken (ng)',
                'description': 'Library Prep: Amount Taken (ng)',
                'min': 0,
                'scale': 'YlGn',
                'format': '{:.,0f}',
                'hidden': amounts_hidden
            }
            report.general_stats_data.append(gsdata)
            report.general_stats_headers.append(gsheaders)
Example #11
0
def matplotlib_linegraph (plotdata, pconfig={}):
    """
    Plot a line graph with Matplot lib and return a HTML string. Either embeds a base64
    encoded image within HTML or writes the plot and links to it. Should be called by
    plot_bargraph, which properly formats the input data.
    """

    # Plot group ID
    if pconfig.get('id') is None:
        pconfig['id'] = 'mqc_mplplot_'+''.join(random.sample(letters, 10))
    # Individual plot IDs
    pids = []
    for k in range(len(plotdata)):
        try:
            name = pconfig['data_labels'][k]['name']
        except:
            name = k+1
        pid = 'mqc_{}_{}'.format(pconfig['id'], name)
        pid = "".join([c for c in pid if c.isalpha() or c.isdigit() or c == '_' or c == '-'])
        pids.append(pid)

    html = '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> ' + \
          'Flat image plot. Toolbox functions such as highlighting / hiding samples will not work ' + \
          '(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>'
    html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig['id'])

    # Same defaults as HighCharts for consistency
    default_colors = ['#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9',
                      '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1']

    # Buttons to cycle through different datasets
    if len(plotdata) > 1 and not config.simple_output:
        html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n'
        for k, p in enumerate(plotdata):
            pid = pids[k]
            active = 'active' if k == 0 else ''
            try:
                name = pconfig['data_labels'][k]['name']
            except:
                name = k+1
            html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format(a=active, pid=pid, n=name)
        html += '</div>\n\n'

    # Go through datasets creating plots
    for pidx, pdata in enumerate(plotdata):

        # Plot ID
        pid = pids[pidx]

        # Save plot data to file
        fdata = OrderedDict()
        lastcats = None
        sharedcats = True
        for d in pdata:
            fdata[d['name']] = OrderedDict()
            for i, x in enumerate(d['data']):
                if type(x) is list:
                    fdata[d['name']][str(x[0])] = x[1]
                    # Check to see if all categories are the same
                    if lastcats is None:
                        lastcats = [x[0] for x in d['data']]
                    elif lastcats != [x[0] for x in d['data']]:
                        sharedcats = False
                else:
                    try:
                        fdata[d['name']][pconfig['categories'][i]] = x
                    except (KeyError, IndexError):
                        fdata[d['name']][str(i)] = x
        # Custom tsv output if the x axis varies
        if not sharedcats and config.data_format == 'tsv':
            fout = ''
            for d in pdata:
                fout += "\t"+"\t".join([str(x[0]) for x in d['data']])
                fout += "\n{}\t".format(d['name'])
                fout += "\t".join([str(x[1]) for x in d['data']])
                fout += "\n"
            with io.open (os.path.join(config.data_dir, '{}.txt'.format(pid)), 'w', encoding='utf-8') as f:
                print( fout.encode('utf-8', 'ignore').decode('utf-8'), file=f )
        else:
            util_functions.write_data_file(fdata, pid)

        # Set up figure
        fig = plt.figure(figsize=(14, 6), frameon=False)
        axes = fig.add_subplot(111)

        # Go through data series
        for idx, d in enumerate(pdata):

            # Default colour index
            cidx = idx
            while cidx >= len(default_colors):
                cidx -= len(default_colors)

            # Line style
            linestyle = 'solid'
            if d.get('dashStyle', None) == 'Dash':
                linestyle = 'dashed'

            # Reformat data (again)
            try:
                axes.plot([x[0] for x in d['data']], [x[1] for x in d['data']], label=d['name'], color=d.get('color', default_colors[cidx]), linestyle=linestyle, linewidth=1, marker=None)
            except TypeError:
                # Categorical data on x axis
                axes.plot(d['data'], label=d['name'], color=d.get('color', default_colors[cidx]), linewidth=1, marker=None)

        # Tidy up axes
        axes.tick_params(labelsize=8, direction='out', left=False, right=False, top=False, bottom=False)
        axes.set_xlabel(pconfig.get('xlab', ''))
        axes.set_ylabel(pconfig.get('ylab', ''))

        # Dataset specific y label
        try:
            axes.set_ylabel(pconfig['data_labels'][pidx]['ylab'])
        except:
            pass

        # Axis limits
        default_ylimits = axes.get_ylim()
        ymin = default_ylimits[0]
        if 'ymin' in pconfig:
            ymin = pconfig['ymin']
        elif 'yCeiling' in pconfig:
            ymin = min(pconfig['yCeiling'], default_ylimits[0])
        ymax = default_ylimits[1]
        if 'ymax' in pconfig:
            ymax = pconfig['ymax']
        elif 'yFloor' in pconfig:
            ymax = max(pconfig['yCeiling'], default_ylimits[1])
        if (ymax - ymin) < pconfig.get('yMinRange', 0):
            ymax = ymin + pconfig['yMinRange']
        axes.set_ylim((ymin, ymax))

        # Dataset specific ymax
        try:
            axes.set_ylim((ymin, pconfig['data_labels'][pidx]['ymax']))
        except:
            pass

        default_xlimits = axes.get_xlim()
        xmin = default_xlimits[0]
        if 'xmin' in pconfig:
            xmin = pconfig['xmin']
        elif 'xCeiling' in pconfig:
            xmin = min(pconfig['xCeiling'], default_xlimits[0])
        xmax = default_xlimits[1]
        if 'xmax' in pconfig:
            xmax = pconfig['xmax']
        elif 'xFloor' in pconfig:
            xmax = max(pconfig['xCeiling'], default_xlimits[1])
        if (xmax - xmin) < pconfig.get('xMinRange', 0):
            xmax = xmin + pconfig['xMinRange']
        axes.set_xlim((xmin, xmax))

        # Plot title
        if 'title' in pconfig:
            plt.text(0.5, 1.05, pconfig['title'], horizontalalignment='center', fontsize=16, transform=axes.transAxes)
        axes.grid(True, zorder=10, which='both', axis='y', linestyle='-', color='#dedede', linewidth=1)

        # X axis categories, if specified
        if 'categories' in pconfig:
            axes.set_xticks([i for i,v in enumerate(pconfig['categories'])])
            axes.set_xticklabels(pconfig['categories'])

        # Axis lines
        xlim = axes.get_xlim()
        axes.plot([xlim[0], xlim[1]], [0, 0], linestyle='-', color='#dedede', linewidth=2)
        axes.set_axisbelow(True)
        axes.spines['right'].set_visible(False)
        axes.spines['top'].set_visible(False)
        axes.spines['bottom'].set_visible(False)
        axes.spines['left'].set_visible(False)

        # Background colours, if specified
        if 'yPlotBands' in pconfig:
            xlim = axes.get_xlim()
            for pb in pconfig['yPlotBands']:
                axes.barh(pb['from'], xlim[1], height = pb['to']-pb['from'], left=xlim[0], color=pb['color'], linewidth=0, zorder=0)
        if 'xPlotBands' in pconfig:
            ylim = axes.get_ylim()
            for pb in pconfig['xPlotBands']:
                axes.bar(pb['from'], ylim[1], width = pb['to']-pb['from'], bottom=ylim[0], color=pb['color'], linewidth=0, zorder=0)

        # Tight layout - makes sure that legend fits in and stuff
        if len(pdata) <= 15:
            lgd = axes.legend(loc='lower center', bbox_to_anchor=(0, -0.22, 1, .102), ncol=5, mode='expand', fontsize=8, frameon=False)
            plt.tight_layout(rect=[0,0.08,1,0.92])
        else:
            plt.tight_layout(rect=[0,0,1,0.92])

        # Should this plot be hidden on report load?
        hidediv = ''
        if pidx > 0:
            hidediv = ' style="display:none;"'

        # Save the plot to the data directory if export is requests
        if config.export_plots:
            for fformat in config.export_plot_formats:
                # Make the directory if it doesn't already exist
                plot_dir = os.path.join(config.plots_dir, fformat)
                if not os.path.exists(plot_dir):
                    os.makedirs(plot_dir)
                # Save the plot
                plot_fn = os.path.join(plot_dir, '{}.{}'.format(pid, fformat))
                fig.savefig(plot_fn, format=fformat, bbox_inches='tight')

        # Output the figure to a base64 encoded string
        if getattr(get_template_mod(), 'base64_plots', True) is True:
            img_buffer = io.BytesIO()
            fig.savefig(img_buffer, format='png', bbox_inches='tight')
            b64_img = base64.b64encode(img_buffer.getvalue()).decode('utf8')
            img_buffer.close()
            html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format(pid, hidediv, b64_img)

        # Save to a file and link <img>
        else:
            plot_relpath = os.path.join(config.plots_dir_name, 'png', '{}.png'.format(pid))
            html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(pid, hidediv, plot_relpath)

        plt.close(fig)


    # Close wrapping div
    html += '</div>'

    report.num_mpl_plots += 1

    return html
Example #12
0
 def write_data_file(self, data, fn, sort_cols=False, data_format=None):
     """ Redirects to report.write_data_file() """
     util_functions.write_data_file(data, fn, sort_cols, data_format)
Example #13
0
 def general_stats_sample_meta(self):
     """ Add metadata about each sample to the General Stats table """
     
     meta = report.ngi.get('sample_meta')
     if meta is not None and len(meta) > 0:
         
         log.info('Found {} samples in StatusDB'.format(len(meta)))
         
         # Write to file
         util_functions.write_data_file(meta, 'ngi_meta')
         
         # Add to General Stats table
         gsdata = dict()
         formats = dict()
         s_names = dict()
         ngi_ids = dict()
         conc_units = ''
         for sid in meta:
             
             # Find first sample name matching this sample ID
             s_name = None
             for x in sorted(self.s_names):
                 if sid in x:
                     s_name = x
                     s_names[s_name] = x
                     ngi_ids[s_name] = sid
                     break
             
             # Skip this sample if we don't have any matching data
             if s_name is None:
                 log.debug("Skipping StatusDB metadata for sample {} as no bioinfo report logs found.".format(sid))
                 continue
             
             # Make a dict to hold new data for General Stats
             gsdata[s_name] = dict()
             
             # NGI name
             try:
                 gsdata[s_name]['user_sample_name'] = report.ngi['ngi_names'][ngi_ids[s_name]]
             except KeyError:
                 pass
             
             # RIN score
             try:
                 gsdata[s_name]['initial_qc_rin'] = meta[sid]['initial_qc']['rin']
             except KeyError:
                 pass
             
             # Try to figure out which library prep was used
             seq_lp = None
             for lp in sorted(meta[sid].get('library_prep', {}).keys()):
                 try:
                     if len(meta[sid]['library_prep'][lp]['sample_run_metrics']) > 0:
                         if seq_lp is None:
                             seq_lp = lp
                         else:
                             seq_lp = None
                             log.warn('Found multiple sequenced lib preps for {} - skipping metadata'.format(sid))
                             break
                 except KeyError:
                     pass
             if seq_lp is not None:
                 try:
                     if meta[sid]['library_prep'][lp]['amount_taken_(ng)'] is not None:
                         gsdata[s_name]['amount_taken'] = meta[sid]['library_prep'][lp]['amount_taken_(ng)']
                 except KeyError:
                     pass
                 try:
                     for lv in sorted(meta[sid]['library_prep'][lp]['library_validation'].keys()):
                         gsdata[s_name]['lp_concentration'] = meta[sid]['library_prep'][lp]['library_validation'][lv]['concentration']
                         formats[s_name] = meta[sid]['library_prep'][lp]['library_validation'][lv]['conc_units']
                 except KeyError:
                     pass
         
         log.info("Matched meta for {} samples from StatusDB with report sample names".format(len(s_names)))
         if len(s_names) == 0:
             return None
         
         # Deal with having more than one initial QC concentration unit
         formats_set = set(formats.values())
         if len(formats_set) > 1:
             log.warning("Mixture of library_validation concentration units! Found: {}".format(", ".join(formats_set)))
             for s_name in gsdata:
                 try:
                     gsdata[s_name]['lp_concentration'] = '{} {}'.format(gsdata[s_name]['lp_concentration'], formats[s_name])
                 except KeyError:
                     pass
         elif len(formats_set) == 1:
             conc_units = formats_set.pop()
         
         # Decide on whether to show or hide conc & amount taken based on range
         conc_hidden = True
         amounts_hidden = True
         try:
             concs = [gsdata[x]['lp_concentration'] for x in gsdata]
             if max(concs) - min(concs) > 50:
                 conc_hidden = False
         except (KeyError, ValueError):
             conc_hidden = False
         try:
             amounts = [gsdata[x]['amount_taken'] for x in gsdata]
             if max(amounts) - min(amounts) > 10:
                 amounts_hidden = False
         except KeyError:
             amounts_hidden = False
         
         # Prepend columns to the General Stats table (far left)
         gsheaders_prepend = OrderedDict()
         gsheaders_prepend['user_sample_name'] = {
             'namespace': 'NGI',
             'title': 'Name',
             'description': 'User sample ID',
             'scale': False
         }
         report.general_stats_data.insert(0, gsdata)
         report.general_stats_headers.insert(0, gsheaders_prepend)
         
         # Add columns to the far right of the General Stats table
         gsheaders = OrderedDict()
         gsheaders['initial_qc_rin'] = {
             'namespace': 'NGI',
             'title': 'RIN',
             'description': 'Initial QC: RNA Integrity Number',
             'min': 0,
             'max': 10,
             'scale': 'YlGn',
             'format': '{:.2f}'
         }
         gsheaders['lp_concentration'] = {
             'namespace': 'NGI',
             'title': 'Lib Conc. ({})'.format(conc_units),
             'description': 'Library Prep: Concentration ({})'.format(conc_units),
             'min': 0,
             'scale': 'YlGn',
             'format': '{:.0f}',
             'hidden': conc_hidden
         }
         gsheaders['amount_taken'] = {
             'namespace': 'NGI',
             'title': 'Amount Taken (ng)',
             'description': 'Library Prep: Amount Taken (ng)',
             'min': 0,
             'scale': 'YlGn',
             'format': '{:.0f}',
             'hidden': amounts_hidden
         }
         report.general_stats_data.append(gsdata)
         report.general_stats_headers.append(gsheaders)
Example #14
0
def matplotlib_linegraph (plotdata, pconfig=None):
    """
    Plot a line graph with Matplot lib and return a HTML string. Either embeds a base64
    encoded image within HTML or writes the plot and links to it. Should be called by
    plot_bargraph, which properly formats the input data.
    """
    if pconfig is None:
        pconfig = {}

    # Plot group ID
    if pconfig.get('id') is None:
        pconfig['id'] = 'mqc_mplplot_'+''.join(random.sample(letters, 10))

    # Sanitise plot ID and check for duplicates
    pconfig['id'] = report.save_htmlid(pconfig['id'])

    # Individual plot IDs
    pids = []
    for k in range(len(plotdata)):
        try:
            name = pconfig['data_labels'][k]['name']
        except:
            name = k+1
        pid = 'mqc_{}_{}'.format(pconfig['id'], name)
        pid = report.save_htmlid(pid)
        pids.append(pid)

    html = '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> ' + \
          'Flat image plot. Toolbox functions such as highlighting / hiding samples will not work ' + \
          '(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>'
    html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig['id'])

    # Same defaults as HighCharts for consistency
    default_colors = ['#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9',
                      '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1']

    # Buttons to cycle through different datasets
    if len(plotdata) > 1 and not config.simple_output:
        html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n'
        for k, p in enumerate(plotdata):
            pid = pids[k]
            active = 'active' if k == 0 else ''
            try:
                name = pconfig['data_labels'][k]['name']
            except:
                name = k+1
            html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format(a=active, pid=pid, n=name)
        html += '</div>\n\n'

    # Go through datasets creating plots
    for pidx, pdata in enumerate(plotdata):

        # Plot ID
        pid = pids[pidx]

        # Save plot data to file
        fdata = OrderedDict()
        lastcats = None
        sharedcats = True
        for d in pdata:
            fdata[d['name']] = OrderedDict()
            for i, x in enumerate(d['data']):
                if type(x) is list:
                    fdata[d['name']][str(x[0])] = x[1]
                    # Check to see if all categories are the same
                    if lastcats is None:
                        lastcats = [x[0] for x in d['data']]
                    elif lastcats != [x[0] for x in d['data']]:
                        sharedcats = False
                else:
                    try:
                        fdata[d['name']][pconfig['categories'][i]] = x
                    except (KeyError, IndexError):
                        fdata[d['name']][str(i)] = x

        # Custom tsv output if the x axis varies
        if not sharedcats and config.data_format == 'tsv':
            fout = ''
            for d in pdata:
                fout += "\t"+"\t".join([str(x[0]) for x in d['data']])
                fout += "\n{}\t".format(d['name'])
                fout += "\t".join([str(x[1]) for x in d['data']])
                fout += "\n"
            with io.open (os.path.join(config.data_dir, '{}.txt'.format(pid)), 'w', encoding='utf-8') as f:
                print( fout.encode('utf-8', 'ignore').decode('utf-8'), file=f )
        else:
            util_functions.write_data_file(fdata, pid)

        # Set up figure
        fig = plt.figure(figsize=(14, 6), frameon=False)
        axes = fig.add_subplot(111)

        # Go through data series
        for idx, d in enumerate(pdata):

            # Default colour index
            cidx = idx
            while cidx >= len(default_colors):
                cidx -= len(default_colors)

            # Line style
            linestyle = 'solid'
            if d.get('dashStyle', None) == 'Dash':
                linestyle = 'dashed'

            # Reformat data (again)
            try:
                axes.plot([x[0] for x in d['data']], [x[1] for x in d['data']], label=d['name'], color=d.get('color', default_colors[cidx]), linestyle=linestyle, linewidth=1, marker=None)
            except TypeError:
                # Categorical data on x axis
                axes.plot(d['data'], label=d['name'], color=d.get('color', default_colors[cidx]), linewidth=1, marker=None)

        # Tidy up axes
        axes.tick_params(labelsize=8, direction='out', left=False, right=False, top=False, bottom=False)
        axes.set_xlabel(pconfig.get('xlab', ''))
        axes.set_ylabel(pconfig.get('ylab', ''))

        # Dataset specific y label
        try:
            axes.set_ylabel(pconfig['data_labels'][pidx]['ylab'])
        except:
            pass

        # Axis limits
        default_ylimits = axes.get_ylim()
        ymin = default_ylimits[0]
        if 'ymin' in pconfig:
            ymin = pconfig['ymin']
        elif 'yCeiling' in pconfig:
            ymin = min(pconfig['yCeiling'], default_ylimits[0])
        ymax = default_ylimits[1]
        if 'ymax' in pconfig:
            ymax = pconfig['ymax']
        elif 'yFloor' in pconfig:
            ymax = max(pconfig['yCeiling'], default_ylimits[1])
        if (ymax - ymin) < pconfig.get('yMinRange', 0):
            ymax = ymin + pconfig['yMinRange']
        axes.set_ylim((ymin, ymax))

        # Dataset specific ymax
        try:
            axes.set_ylim((ymin, pconfig['data_labels'][pidx]['ymax']))
        except:
            pass

        default_xlimits = axes.get_xlim()
        xmin = default_xlimits[0]
        if 'xmin' in pconfig:
            xmin = pconfig['xmin']
        elif 'xCeiling' in pconfig:
            xmin = min(pconfig['xCeiling'], default_xlimits[0])
        xmax = default_xlimits[1]
        if 'xmax' in pconfig:
            xmax = pconfig['xmax']
        elif 'xFloor' in pconfig:
            xmax = max(pconfig['xCeiling'], default_xlimits[1])
        if (xmax - xmin) < pconfig.get('xMinRange', 0):
            xmax = xmin + pconfig['xMinRange']
        axes.set_xlim((xmin, xmax))

        # Plot title
        if 'title' in pconfig:
            plt.text(0.5, 1.05, pconfig['title'], horizontalalignment='center', fontsize=16, transform=axes.transAxes)
        axes.grid(True, zorder=10, which='both', axis='y', linestyle='-', color='#dedede', linewidth=1)

        # X axis categories, if specified
        if 'categories' in pconfig:
            axes.set_xticks([i for i,v in enumerate(pconfig['categories'])])
            axes.set_xticklabels(pconfig['categories'])

        # Axis lines
        xlim = axes.get_xlim()
        axes.plot([xlim[0], xlim[1]], [0, 0], linestyle='-', color='#dedede', linewidth=2)
        axes.set_axisbelow(True)
        axes.spines['right'].set_visible(False)
        axes.spines['top'].set_visible(False)
        axes.spines['bottom'].set_visible(False)
        axes.spines['left'].set_visible(False)

        # Background colours, if specified
        if 'yPlotBands' in pconfig:
            xlim = axes.get_xlim()
            for pb in pconfig['yPlotBands']:
                axes.barh(pb['from'], xlim[1], height = pb['to']-pb['from'], left=xlim[0], color=pb['color'], linewidth=0, zorder=0)
        if 'xPlotBands' in pconfig:
            ylim = axes.get_ylim()
            for pb in pconfig['xPlotBands']:
                axes.bar(pb['from'], ylim[1], width = pb['to']-pb['from'], bottom=ylim[0], color=pb['color'], linewidth=0, zorder=0)

        # Tight layout - makes sure that legend fits in and stuff
        if len(pdata) <= 15:
            axes.legend(loc='lower center', bbox_to_anchor=(0, -0.22, 1, .102), ncol=5, mode='expand', fontsize=8, frameon=False)
            plt.tight_layout(rect=[0,0.08,1,0.92])
        else:
            plt.tight_layout(rect=[0,0,1,0.92])

        # Should this plot be hidden on report load?
        hidediv = ''
        if pidx > 0:
            hidediv = ' style="display:none;"'

        # Save the plot to the data directory if export is requests
        if config.export_plots:
            for fformat in config.export_plot_formats:
                # Make the directory if it doesn't already exist
                plot_dir = os.path.join(config.plots_dir, fformat)
                if not os.path.exists(plot_dir):
                    os.makedirs(plot_dir)
                # Save the plot
                plot_fn = os.path.join(plot_dir, '{}.{}'.format(pid, fformat))
                fig.savefig(plot_fn, format=fformat, bbox_inches='tight')

        # Output the figure to a base64 encoded string
        if getattr(get_template_mod(), 'base64_plots', True) is True:
            img_buffer = io.BytesIO()
            fig.savefig(img_buffer, format='png', bbox_inches='tight')
            b64_img = base64.b64encode(img_buffer.getvalue()).decode('utf8')
            img_buffer.close()
            html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format(pid, hidediv, b64_img)

        # Save to a file and link <img>
        else:
            plot_relpath = os.path.join(config.plots_dir_name, 'png', '{}.png'.format(pid))
            html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(pid, hidediv, plot_relpath)

        plt.close(fig)


    # Close wrapping div
    html += '</div>'

    report.num_mpl_plots += 1

    return html
Example #15
0
def make_table (dt):
    """
    Build the HTML needed for a MultiQC table.
    :param data: MultiQC datatable object
    """

    table_id = dt.pconfig.get('id', 'table_{}'.format(''.join(random.sample(letters, 4))) )
    table_id = report.save_htmlid(table_id)
    t_headers = OrderedDict()
    t_modal_headers = OrderedDict()
    t_rows = OrderedDict()
    dt.raw_vals = defaultdict(lambda: dict())
    empty_cells = dict()
    hidden_cols = 1
    table_title = dt.pconfig.get('table_title')
    if table_title is None:
        table_title = table_id.replace("_", " ").title()

    for idx, k, header in dt.get_headers_in_order():

        rid = header['rid']

        # Build the table header cell
        shared_key = ''
        if header.get('shared_key', None) is not None:
            shared_key = ' data-shared-key={}'.format(header['shared_key'])

        hide = ''
        muted = ''
        checked = ' checked="checked"'
        if header.get('hidden', False) is True:
            hide = 'hidden'
            muted = ' text-muted'
            checked = ''
            hidden_cols += 1

        data_attr = 'data-dmax="{}" data-dmin="{}" data-namespace="{}" {}' \
            .format(header['dmax'], header['dmin'], header['namespace'], shared_key)

        cell_contents = '<span class="mqc_table_tooltip" title="{}: {}">{}</span>' \
            .format(header['namespace'], header['description'], header['title'])

        t_headers[rid] = '<th id="header_{rid}" class="{rid} {h}" {da}>{c}</th>' \
            .format(rid=rid, h=hide, da=data_attr, c=cell_contents)

        empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format(rid=rid, h=hide)

        # Build the modal table row
        t_modal_headers[rid] = """
        <tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);">
          <td class="sorthandle ui-sortable-handle">||</span></td>
          <td style="text-align:center;">
            <input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}">
          </td>
          <td>{name}</td>
          <td>{title}</td>
          <td>{desc}</td>
          <td>{col_id}</td>
          <td>{sk}</td>
        </tr>""".format(
                rid = rid,
                muted = muted,
                checked = checked,
                tid = table_id,
                col = header['colour'],
                name = header['namespace'],
                title = header['title'],
                desc = header['description'],
                col_id = '<code>{}</code>'.format(k),
                sk = header.get('shared_key', '')
            )

        # Make a colour scale
        if header['scale'] == False:
            c_scale = None
        else:
            c_scale = mqc_colour.mqc_colour_scale(header['scale'], header['dmin'], header['dmax'])

        # Add the data table cells
        for (s_name, samp) in dt.data[idx].items():
            if k in samp:
                val = samp[k]
                kname = '{}_{}'.format(header['namespace'], rid)
                dt.raw_vals[s_name][kname] = val

                if 'modify' in header and callable(header['modify']):
                    val = header['modify'](val)

                try:
                    dmin = header['dmin']
                    dmax = header['dmax']
                    percentage = ((float(val) - dmin) / (dmax - dmin)) * 100
                    percentage = min(percentage, 100)
                    percentage = max(percentage, 0)
                except (ZeroDivisionError,ValueError):
                    percentage = 0

                try:
                    valstring = str(header['format'].format(val))
                except ValueError:
                    try:
                        valstring = str(header['format'].format(float(val)))
                    except ValueError:
                        valstring = str(val)
                except:
                    valstring = str(val)

                # This is horrible, but Python locale settings are worse
                if config.thousandsSep_format is None:
                    config.thousandsSep_format = '<span class="mqc_thousandSep"></span>'
                if config.decimalPoint_format is None:
                    config.decimalPoint_format = '.'
                valstring = valstring.replace('.', 'DECIMAL').replace(',', 'THOUSAND')
                valstring = valstring.replace('DECIMAL', config.decimalPoint_format).replace('THOUSAND', config.thousandsSep_format)

                # Percentage suffixes etc
                valstring += header.get('suffix', '')

                # Conditional formatting
                cmatches = { cfck: False for cfc in config.table_cond_formatting_colours for cfck in cfc }
                # Find general rules followed by column-specific rules
                for cfk in ['all_columns', rid]:
                    if cfk in config.table_cond_formatting_rules:
                        # Loop through match types
                        for ftype in cmatches.keys():
                            # Loop through array of comparison types
                            for cmp in config.table_cond_formatting_rules[cfk].get(ftype, []):
                                try:
                                    # Each comparison should be a dict with single key: val
                                    if 's_eq' in cmp and str(cmp['s_eq']).lower() == str(val).lower():
                                        cmatches[ftype] = True
                                    if 's_contains' in cmp and str(cmp['s_contains']).lower() in str(val).lower():
                                        cmatches[ftype] = True
                                    if 's_ne' in cmp and str(cmp['s_ne']).lower() != str(val).lower():
                                        cmatches[ftype] = True
                                    if 'eq' in cmp and float(cmp['eq']) == float(val):
                                        cmatches[ftype] = True
                                    if 'ne' in cmp and float(cmp['ne']) != float(val):
                                        cmatches[ftype] = True
                                    if 'gt' in cmp and float(cmp['gt']) < float(val):
                                        cmatches[ftype] = True
                                    if 'lt' in cmp and float(cmp['lt']) > float(val):
                                        cmatches[ftype] = True
                                except:
                                    logger.warn("Not able to apply table conditional formatting to '{}' ({})".format(val, cmp))
                # Apply HTML in order of config keys
                bgcol = None
                for cfc in config.table_cond_formatting_colours:
                    for cfck in cfc: # should always be one, but you never know
                        if cmatches[cfck]:
                            bgcol = cfc[cfck]
                if bgcol is not None:
                    valstring = '<span class="badge" style="background-color:{}">{}</span>'.format(bgcol, valstring)

                # Build HTML
                if not header['scale']:
                    if s_name not in t_rows:
                        t_rows[s_name] = dict()
                    t_rows[s_name][rid] = '<td class="{rid} {h}">{v}</td>'.format(rid=rid, h=hide, v=valstring)
                else:
                    if c_scale is not None:
                        col = ' background-color:{};'.format(c_scale.get_colour(val))
                    else:
                        col = ''
                    bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(percentage, col)
                    val_html = '<span class="val">{}</span>'.format(valstring)
                    wrapper_html = '<div class="wrapper">{}{}</div>'.format(bar_html, val_html)

                    if s_name not in t_rows:
                        t_rows[s_name] = dict()
                    t_rows[s_name][rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(rid=rid, h=hide, c=wrapper_html)

        # Remove header if we don't have any filled cells for it
        if sum([len(rows) for rows in t_rows.values()]) == 0:
            t_headers.pop(rid, None)
            t_modal_headers.pop(rid, None)
            logger.debug('Removing header {} from general stats table, as no data'.format(k))

    #
    # Put everything together
    #

    # Buttons above the table
    html = ''
    if not config.simple_output:

        # Copy Table Button
        html += """
        <button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}">
            <span class="glyphicon glyphicon-copy"></span> Copy table
        </button>
        """.format(tid=table_id)

        # Configure Columns Button
        if len(t_headers) > 1:
            html += """
            <button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal">
                <span class="glyphicon glyphicon-th"></span> Configure Columns
            </button>
            """.format(tid=table_id)

        # Sort By Highlight button
        html += """
        <button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;">
            <span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight
        </button>
        """.format(tid=table_id)

        # Scatter Plot Button
        if len(t_headers) > 1:
            html += """
            <button type="button" class="mqc_table_makeScatter btn btn-default btn-sm" data-toggle="modal" data-target="#tableScatterModal" data-table="#{tid}">
                <span class="glyphicon glyphicon glyphicon-stats"></span> Plot
            </button>
            """.format(tid=table_id)

        # "Showing x of y columns" text
        html += """
        <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small>
        """.format(tid=table_id, nrows=len(t_rows), ncols_vis = (len(t_headers)+1)-hidden_cols, ncols=len(t_headers))

    # Build the table itself
    collapse_class = 'mqc-table-collapse' if len(t_rows) > 10 and config.collapse_tables else ''
    html += """
        <div id="{tid}_container" class="mqc_table_container">
            <div class="table-responsive mqc-table-responsive {cc}">
                <table id="{tid}" class="table table-condensed mqc_table" data-title="{title}">
        """.format( tid=table_id, title=table_title, cc=collapse_class)

    # Build the header row
    col1_header = dt.pconfig.get('col1_header', 'Sample Name')
    html += '<thead><tr><th class="rowheader">{}</th>{}</tr></thead>'.format(col1_header, ''.join(t_headers.values()))

    # Build the table body
    html += '<tbody>'
    t_row_keys = t_rows.keys()
    if dt.pconfig.get('sortRows') is not False:
        t_row_keys = sorted(t_row_keys)
    for s_name in t_row_keys:
        html += '<tr>'
        # Sample name row header
        html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format(sn=s_name)
        for k in t_headers:
            html += t_rows[s_name].get(k, empty_cells[k])
        html += '</tr>'
    html += '</tbody></table></div>'
    if len(t_rows) > 10 and config.collapse_tables:
        html += '<div class="mqc-table-expand"><span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span></div>'
    html += '</div>'

    # Build the bootstrap modal to customise columns and order
    if not config.simple_output:
        html += """
    <!-- MultiQC Table Columns Modal -->
    <div class="modal fade" id="{tid}_configModal" tabindex="-1">
      <div class="modal-dialog modal-lg">
        <div class="modal-content">
          <div class="modal-header">
            <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
            <h4 class="modal-title">{title}: Columns</h4>
          </div>
          <div class="modal-body">
            <p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p>
            <p>
                <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button>
                <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button>
            </p>
            <table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table" data-title="{title}">
              <thead>
                <tr>
                  <th class="sorthandle" style="text-align:center;">Sort</th>
                  <th style="text-align:center;">Visible</th>
                  <th>Group</th>
                  <th>Column</th>
                  <th>Description</th>
                  <th>ID</th>
                  <th>Scale</th>
                </tr>
              </thead>
              <tbody>
                {trows}
              </tbody>
            </table>
        </div>
        <div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div>
    </div> </div> </div>""".format( tid=table_id, title=table_title, trows=''.join(t_modal_headers.values()) )

    # Save the raw values to a file if requested
    if dt.pconfig.get('save_file') is True:
        fn = dt.pconfig.get('raw_data_fn', 'multiqc_{}'.format(table_id) )
        util_functions.write_data_file(dt.raw_vals, fn )
        report.saved_raw_data[fn] = dt.raw_vals

    return html
Example #16
0
def make_table(dt):
    """
    Build the HTML needed for a MultiQC table.
    :param data: MultiQC datatable object
    """

    table_id = dt.pconfig.get(
        'id', 'table_{}'.format(''.join(random.sample(letters, 4))))
    table_id = report.save_htmlid(table_id)
    t_headers = OrderedDict()
    t_modal_headers = OrderedDict()
    t_rows = OrderedDict()
    dt.raw_vals = defaultdict(lambda: dict())
    empty_cells = dict()
    hidden_cols = 1
    table_title = dt.pconfig.get('table_title')
    if table_title is None:
        table_title = table_id.replace("_", " ").title()

    for idx, k, header in dt.get_headers_in_order():

        rid = header['rid']

        # Build the table header cell
        shared_key = ''
        if header.get('shared_key', None) is not None:
            shared_key = ' data-shared-key={}'.format(header['shared_key'])

        hide = ''
        muted = ''
        checked = ' checked="checked"'
        if header.get('hidden', False) is True:
            hide = 'hidden'
            muted = ' text-muted'
            checked = ''
            hidden_cols += 1

        data_attr = 'data-dmax="{}" data-dmin="{}" data-namespace="{}" {}' \
            .format(header['dmax'], header['dmin'], header['namespace'], shared_key)

        cell_contents = '<span class="mqc_table_tooltip" title="{}: {}">{}</span>' \
            .format(header['namespace'], header['description'], header['title'])

        t_headers[rid] = '<th id="header_{rid}" class="{rid} {h}" {da}>{c}</th>' \
            .format(rid=rid, h=hide, da=data_attr, c=cell_contents)

        empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format(
            rid=rid, h=hide)

        # Build the modal table row
        t_modal_headers[rid] = """
        <tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);">
          <td class="sorthandle ui-sortable-handle">||</span></td>
          <td style="text-align:center;">
            <input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}">
          </td>
          <td>{name}</td>
          <td>{title}</td>
          <td>{desc}</td>
          <td>{col_id}</td>
          <td>{sk}</td>
        </tr>""".format(rid=rid,
                        muted=muted,
                        checked=checked,
                        tid=table_id,
                        col=header['colour'],
                        name=header['namespace'],
                        title=header['title'],
                        desc=header['description'],
                        col_id='<code>{}</code>'.format(k),
                        sk=header.get('shared_key', ''))

        # Make a colour scale
        if header['scale'] == False:
            c_scale = None
        else:
            c_scale = mqc_colour.mqc_colour_scale(header['scale'],
                                                  header['dmin'],
                                                  header['dmax'])

        # Add the data table cells
        for (s_name, samp) in dt.data[idx].items():
            if k in samp:
                val = samp[k]
                kname = '{}_{}'.format(header['namespace'], rid)
                dt.raw_vals[s_name][kname] = val

                if 'modify' in header and callable(header['modify']):
                    val = header['modify'](val)

                try:
                    dmin = header['dmin']
                    dmax = header['dmax']
                    percentage = ((float(val) - dmin) / (dmax - dmin)) * 100
                    percentage = min(percentage, 100)
                    percentage = max(percentage, 0)
                except (ZeroDivisionError, ValueError):
                    percentage = 0

                try:
                    valstring = str(header['format'].format(val))
                except ValueError:
                    try:
                        valstring = str(header['format'].format(float(val)))
                    except ValueError:
                        valstring = str(val)
                except:
                    valstring = str(val)

                # This is horrible, but Python locale settings are worse
                if config.thousandsSep_format is None:
                    config.thousandsSep_format = '<span class="mqc_thousandSep"></span>'
                if config.decimalPoint_format is None:
                    config.decimalPoint_format = '.'
                valstring = valstring.replace('.', 'DECIMAL').replace(
                    ',', 'THOUSAND')
                valstring = valstring.replace(
                    'DECIMAL', config.decimalPoint_format).replace(
                        'THOUSAND', config.thousandsSep_format)

                # Percentage suffixes etc
                valstring += header.get('suffix', '')
                # Build HTML
                if s_name not in t_rows:
                    t_rows[s_name] = dict()
                t_rows[s_name][rid] = '<td class="{rid} {h}">{v}</td>'.format(
                    rid=rid, h=hide, v=valstring)
                # else:
                #     if c_scale is not None:
                #         col = ' background-color:{};'.format(c_scale.get_colour(val))
                #     else:
                #         col = ''
                #     bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(percentage, col)
                #     val_html = '<span class="val">{}</span>'.format(valstring)
                #     wrapper_html = '<div class="wrapper">{}{}</div>'.format(bar_html, val_html)
                #
                #     if s_name not in t_rows:
                #         t_rows[s_name] = dict()
                #     t_rows[s_name][rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(rid=rid, h=hide, c=wrapper_html)

        # Remove header if we don't have any filled cells for it
        if sum([len(rows) for rows in t_rows.values()]) == 0:
            t_headers.pop(rid, None)
            t_modal_headers.pop(rid, None)
            logger.debug(
                'Removing header {} from general stats table, as no data'.
                format(k))

    #
    # Put everything together
    #

    # Buttons above the table
    html = ''
    if not config.simple_output:

        # Copy Table Button
        html += """
        <button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}">
            <span class="glyphicon glyphicon-copy"></span> Copy table
        </button>
        """.format(tid=table_id)

        # Configure Columns Button
        if len(t_headers) > 1:
            html += """
            <button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal">
                <span class="glyphicon glyphicon-th"></span> Configure Columns
            </button>
            """.format(tid=table_id)

        # Sort By Highlight button
        html += """
        <button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;">
            <span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight
        </button>
        """.format(tid=table_id)

        # Scatter Plot Button
        if len(t_headers) > 1:
            html += """
            <button type="button" class="mqc_table_makeScatter btn btn-default btn-sm" data-toggle="modal" data-target="#tableScatterModal" data-table="#{tid}">
                <span class="glyphicon glyphicon glyphicon-stats"></span> Plot
            </button>
            """.format(tid=table_id)

        # "Showing x of y columns" text
        html += """
        <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small>
        """.format(tid=table_id,
                   nrows=len(t_rows),
                   ncols_vis=(len(t_headers) + 1) - hidden_cols,
                   ncols=len(t_headers))

    # Build the table itself
    collapse_class = 'mqc-table-collapse' if len(
        t_rows) > 10 and config.collapse_tables else ''
    html += """
        <div id="{tid}_container" class="mqc_table_container">
            <div class="table-responsive mqc-table-responsive {cc}">
                <table id="{tid}" class="table table-condensed mqc_table" data-title="{title}">
        """.format(tid=table_id, title=table_title, cc=collapse_class)

    # Build the header row
    col1_header = dt.pconfig.get('col1_header', ' ')
    html += '<thead><tr><th class="rowheader">{}</th>{}</tr></thead>'.format(
        col1_header, ''.join(t_headers.values()))

    # Build the table body
    html += '<tbody>'
    t_row_keys = t_rows.keys()
    if dt.pconfig.get('sortRows') is not False:
        t_row_keys = sorted(t_row_keys)
    for s_name in t_row_keys:
        html += '<tr>'
        # Sample name row header
        html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format(
            sn=s_name)
        for k in t_headers:
            html += t_rows[s_name].get(k, empty_cells[k])
        html += '</tr>'
    html += '</tbody></table></div>'
    if len(t_rows) > 10 and config.collapse_tables:
        html += '<div class="mqc-table-expand"><span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span></div>'
    html += '</div>'

    # Build the bootstrap modal to customise columns and order
    if not config.simple_output:
        html += """
    <!-- MultiQC Table Columns Modal -->
    <div class="modal fade" id="{tid}_configModal" tabindex="-1">
      <div class="modal-dialog modal-lg">
        <div class="modal-content">
          <div class="modal-header">
            <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
            <h4 class="modal-title">{title}: Columns</h4>
          </div>
          <div class="modal-body">
            <p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p>
            <p>
                <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button>
                <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button>
            </p>
            <table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table" data-title="{title}">
              <thead>
                <tr>
                  <th class="sorthandle" style="text-align:center;">Sort</th>
                  <th style="text-align:center;">Visible</th>
                  <th>Group</th>
                  <th>Column</th>
                  <th>Description</th>
                  <th>ID</th>
                  <th>Scale</th>
                </tr>
              </thead>
              <tbody>
                {trows}
              </tbody>
            </table>
        </div>
        <div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div>
    </div> </div> </div>""".format(tid=table_id,
                                   title=table_title,
                                   trows=''.join(t_modal_headers.values()))

    # Save the raw values to a file if requested
    if dt.pconfig.get('save_file') is True:
        fn = dt.pconfig.get('raw_data_fn', 'multiqc_{}'.format(table_id))
        util_functions.write_data_file(dt.raw_vals, fn)
        report.saved_raw_data[fn] = dt.raw_vals

    return html
Example #17
0
def make_table (dt):
    """
    Build the HTML needed for a MultiQC table.
    :param data: MultiQC datatable object
    """
    
    table_id = dt.pconfig.get('id', 'table_{}'.format(''.join(random.sample(letters, 4))) )
    t_headers = OrderedDict()
    t_modal_headers = OrderedDict()
    t_rows = defaultdict(lambda: dict())
    dt.raw_vals = defaultdict(lambda: dict())
    empty_cells = dict()
    hidden_cols = 1
    
    for idx, hs in enumerate(dt.headers):
        for k, header in hs.items():
            
            rid = header['rid']
            
            # Build the table header cell
            shared_key = ''
            if header.get('shared_key', None) is not None:
                shared_key = ' data-shared-key={}'.format(header['shared_key'])
                
            hide = ''
            muted = ''
            checked = ' checked="checked"'
            if header.get('hidden', False) is True:
                hide = 'hidden'
                muted = ' text-muted'
                checked = ''
                hidden_cols += 1
            
            data_attr = 'data-chroma-scale="{}" data-chroma-max="{}" data-chroma-min="{}" {}' \
                .format(header['scale'], header['dmax'], header['dmin'], shared_key)
            
            cell_contents = '<span data-toggle="tooltip" title="{}: {}">{}</span>' \
                .format(header['namespace'], header['description'], header['title'])
            
            t_headers[rid] = '<th id="header_{rid}" class="chroma-col {rid} {h}" {d}>{c}</th>' \
                .format(rid=rid, d=data_attr, h=hide, c=cell_contents)
            
            empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format(rid=rid, h=hide)
            
            # Build the modal table row
            t_modal_headers[rid] = """
            <tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);">
              <td class="sorthandle ui-sortable-handle">||</span></td>
              <td style="text-align:center;">
                <input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}">
              </td>
              <td>{name}</td>
              <td>{title}</td>
              <td>{desc}</td>
              <td>{col_id}</td>
              <td>{sk}</td>
            </tr>""".format(
                    rid = rid,
                    muted = muted,
                    checked = checked,
                    tid = table_id,
                    col = header['colour'],
                    name = header['namespace'],
                    title = header['title'],
                    desc = header['description'],
                    col_id = '<code>{}</code>'.format(k),
                    sk = header.get('shared_key', '')
                )
            
            # Add the data table cells
            for (s_name, samp) in dt.data[idx].items():
                if k in samp:
                    val = samp[k]
                    dt.raw_vals[s_name][rid] = val
                    
                    if 'modify' in header and callable(header['modify']):
                        val = header['modify'](val)
                    
                    try:
                        dmin = header['dmin']
                        dmax = header['dmax']
                        percentage = ((float(val) - dmin) / (dmax - dmin)) * 100;
                        percentage = min(percentage, 100)
                        percentage = max(percentage, 0)
                    except (ZeroDivisionError,ValueError):
                        percentage = 0
                    
                    try:
                        val = header['format'].format(val)
                    except ValueError:
                        try:
                            val = header['format'].format(float(samp[k]))
                        except ValueError:
                            val = samp[k]
                    except:
                        val = samp[k]
                    
                    # Build HTML
                    bar_html = '<span class="bar" style="width:{}%;"></span>'.format(percentage)
                    val_html = '<span class="val">{}</span>'.format(val)
                    wrapper_html = '<div class="wrapper">{}{}</div>'.format(bar_html, val_html)
                    
                    t_rows[s_name][rid] = \
                        '<td class="data-coloured {rid} {h}">{c}</td>'.format(rid=rid, h=hide, c=wrapper_html)
            
            # Remove header if we don't have any filled cells for it
            if sum([len(rows) for rows in t_rows.values()]) == 0:
                t_headers.pop(rid, None)
                t_modal_headers.pop(rid, None)
                logger.debug('Removing header {} from general stats table, as no data'.format(k))
    
    #
    # Put everything together
    #
    
    # Buttons above the table
    html = """
        <button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}">
            <span class="glyphicon glyphicon-copy"></span> Copy table
        </button>
        <button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal">
            <span class="glyphicon glyphicon-th"></span> Configure Columns
        </button>
        <button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;">
            <span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight
        </button>
        <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small>
    """.format(tid=table_id, nrows=len(t_rows), ncols_vis = len(t_headers)-hidden_cols, ncols=len(t_headers))
    
    # Build the table itself
    html += """
        <div id="{tid}_container" class="mqc_table_container">
            <div class="table-responsive">
                <table id="{tid}" class="table table-condensed mqc_table">
        """.format(tid=table_id)
    
    # Build the header row
    html += '<thead><tr><th class="rowheader">Sample Name</th>{}</tr></thead>'.format(''.join(t_headers.values()))
    
    # Build the table body
    html += '<tbody>'
    for s_name in sorted(t_rows.keys()):
        html += '<tr>'
        # Sample name row header
        html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format(sn=s_name)
        for k in t_headers:
            html += t_rows[s_name].get(k, empty_cells[k])
        html += '</tr>'
    html += '</tbody></table></div></div>'
    
    # Build the bootstrap modal to customise columns and order
    html += """
    <!-- MultiQC Table Columns Modal -->
    <div class="modal fade" id="{tid}_configModal" tabindex="-1">
      <div class="modal-dialog modal-lg">
        <div class="modal-content">
          <div class="modal-header">
            <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
            <h4 class="modal-title">{title}: Columns</h4>
          </div>
          <div class="modal-body">
            <p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p>
            <p>
                <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button>
                <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button>
            </p>
            <table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table">
              <thead>
                <tr>
                  <th class="sorthandle" style="text-align:center;">Sort</th>
                  <th style="text-align:center;">Visible</th>
                  <th>Group</th>
                  <th>Column</th>
                  <th>Description</th>
                  <th>ID</th>
                  <th>Scale</th>
                </tr>
              </thead>
              <tbody>
                {trows}
              </tbody>
            </table>
        </div>
        <div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div>
    </div> </div> </div>""".format( tid=table_id, title=dt.pconfig.get('table_title', table_id), trows=''.join(t_modal_headers.values()) )
    
    # Save the raw values to a file if requested
    if dt.pconfig.get('save_file') is True:
        fn = dt.pconfig.get('raw_data_fn', 'multiqc_{}'.format(table_id) )
        util_functions.write_data_file(dt.raw_vals, fn )
    
    return html
    
    
    
    
    
    
Example #18
0
 def write_data_file(self, data, fn, sort_cols=False, data_format=None):
     """ Saves raw data to a dictionary for downstream use, then redirects
     to report.write_data_file() to create the file in the report directory """
     report.saved_raw_data[fn] = data
     util_functions.write_data_file(data, fn, sort_cols, data_format)