def write_data_file(self, data, fn, sort_cols=False, data_format=None): """ Saves raw data to a dictionary for downstream use, then redirects to report.write_data_file() to create the file in the report directory """ i = 1 base_fn = fn while fn in report.saved_raw_data: fn = '{}_{}'.format(base_fn, i) report.saved_raw_data[fn] = data util_functions.write_data_file(data, fn, sort_cols, data_format)
def write_data_file(self, data, fn, sort_cols=False, data_format=None): """Saves raw data to a dictionary for downstream use, then redirects to report.write_data_file() to create the file in the report directory""" # Append custom module anchor if set mod_cust_config = getattr(self, "mod_cust_config", {}) if "anchor" in mod_cust_config: fn = "{}_{}".format(fn, mod_cust_config["anchor"]) # Generate a unique filename if the file already exists (running module multiple times) i = 1 base_fn = fn while fn in report.saved_raw_data: fn = "{}_{}".format(base_fn, i) i += 1 # Save the file report.saved_raw_data[fn] = data util_functions.write_data_file(data, fn, sort_cols, data_format)
def make_table(dt): """ Build the HTML needed for a MultiQC table. :param data: MultiQC datatable object """ table_id = dt.pconfig.get( "id", "table_{}".format("".join(random.sample(letters, 4)))) table_id = report.save_htmlid(table_id) t_headers = OrderedDict() t_modal_headers = OrderedDict() t_rows = OrderedDict() t_rows_empty = OrderedDict() dt.raw_vals = defaultdict(lambda: dict()) empty_cells = dict() hidden_cols = 1 table_title = dt.pconfig.get("table_title") if table_title is None: table_title = table_id.replace("_", " ").title() for idx, k, header in dt.get_headers_in_order(): rid = header["rid"] # Build the table header cell shared_key = "" if header.get("shared_key", None) is not None: shared_key = " data-shared-key={}".format(header["shared_key"]) hide = "" muted = "" checked = ' checked="checked"' if header.get("hidden", False) is True: hide = "hidden" muted = " text-muted" checked = "" hidden_cols += 1 data_attr = 'data-dmax="{}" data-dmin="{}" data-namespace="{}" {}'.format( header["dmax"], header["dmin"], header["namespace"], shared_key) cell_contents = '<span class="mqc_table_tooltip" title="{}: {}">{}</span>'.format( header["namespace"], header["description"], header["title"]) t_headers[ rid] = '<th id="header_{rid}" class="{rid} {h}" {da}>{c}</th>'.format( rid=rid, h=hide, da=data_attr, c=cell_contents) empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format( rid=rid, h=hide) # Build the modal table row t_modal_headers[rid] = """ <tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);"> <td class="sorthandle ui-sortable-handle">||</span></td> <td style="text-align:center;"> <input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}"> </td> <td>{name}</td> <td>{title}</td> <td>{desc}</td> <td>{col_id}</td> <td>{sk}</td> </tr>""".format( rid=rid, muted=muted, checked=checked, tid=table_id, col=header["colour"], name=header["namespace"], title=header["title"], desc=header["description"], col_id="<code>{}</code>".format(k), sk=header.get("shared_key", ""), ) # Make a colour scale if header["scale"] == False: c_scale = None else: c_scale = mqc_colour.mqc_colour_scale(header["scale"], header["dmin"], header["dmax"]) # Add the data table cells for (s_name, samp) in dt.data[idx].items(): if k in samp: val = samp[k] kname = "{}_{}".format(header["namespace"], rid) dt.raw_vals[s_name][kname] = val if "modify" in header and callable(header["modify"]): val = header["modify"](val) try: dmin = header["dmin"] dmax = header["dmax"] percentage = ((float(val) - dmin) / (dmax - dmin)) * 100 percentage = min(percentage, 100) percentage = max(percentage, 0) except (ZeroDivisionError, ValueError): percentage = 0 try: valstring = str(header["format"].format(val)) except ValueError: try: valstring = str(header["format"].format(float(val))) except ValueError: valstring = str(val) except: valstring = str(val) # This is horrible, but Python locale settings are worse if config.thousandsSep_format is None: config.thousandsSep_format = '<span class="mqc_thousandSep"></span>' if config.decimalPoint_format is None: config.decimalPoint_format = "." valstring = valstring.replace(".", "DECIMAL").replace( ",", "THOUSAND") valstring = valstring.replace( "DECIMAL", config.decimalPoint_format).replace( "THOUSAND", config.thousandsSep_format) # Percentage suffixes etc valstring += header.get("suffix", "") # Conditional formatting cmatches = { cfck: False for cfc in config.table_cond_formatting_colours for cfck in cfc } # Find general rules followed by column-specific rules for cfk in ["all_columns", rid]: if cfk in config.table_cond_formatting_rules: # Loop through match types for ftype in cmatches.keys(): # Loop through array of comparison types for cmp in config.table_cond_formatting_rules[ cfk].get(ftype, []): try: # Each comparison should be a dict with single key: val if "s_eq" in cmp and str( cmp["s_eq"]).lower() == str( val).lower(): cmatches[ftype] = True if "s_contains" in cmp and str( cmp["s_contains"]).lower() in str( val).lower(): cmatches[ftype] = True if "s_ne" in cmp and str( cmp["s_ne"]).lower() != str( val).lower(): cmatches[ftype] = True if "eq" in cmp and float( cmp["eq"]) == float(val): cmatches[ftype] = True if "ne" in cmp and float( cmp["ne"]) != float(val): cmatches[ftype] = True if "gt" in cmp and float( cmp["gt"]) < float(val): cmatches[ftype] = True if "lt" in cmp and float( cmp["lt"]) > float(val): cmatches[ftype] = True except: logger.warning( "Not able to apply table conditional formatting to '{}' ({})" .format(val, cmp)) # Apply HTML in order of config keys bgcol = None for cfc in config.table_cond_formatting_colours: for cfck in cfc: # should always be one, but you never know if cmatches[cfck]: bgcol = cfc[cfck] if bgcol is not None: valstring = '<span class="badge" style="background-color:{}">{}</span>'.format( bgcol, valstring) # Build HTML if not header["scale"]: if s_name not in t_rows: t_rows[s_name] = dict() t_rows[s_name][ rid] = '<td class="{rid} {h}">{v}</td>'.format( rid=rid, h=hide, v=valstring) else: if c_scale is not None: col = " background-color:{};".format( c_scale.get_colour(val)) else: col = "" bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format( percentage, col) val_html = '<span class="val">{}</span>'.format(valstring) wrapper_html = '<div class="wrapper">{}{}</div>'.format( bar_html, val_html) if s_name not in t_rows: t_rows[s_name] = dict() t_rows[s_name][ rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format( rid=rid, h=hide, c=wrapper_html) # Is this cell hidden or empty? if s_name not in t_rows_empty: t_rows_empty[s_name] = dict() t_rows_empty[s_name][rid] = header.get( "hidden", False) or str(val).strip() == "" # Remove header if we don't have any filled cells for it if sum([len(rows) for rows in t_rows.values()]) == 0: if header.get("hidden", False) is True: hidden_cols -= 1 t_headers.pop(rid, None) t_modal_headers.pop(rid, None) logger.debug("Removing header {} from table, as no data".format(k)) # # Put everything together # # Buttons above the table html = "" if not config.simple_output: # Copy Table Button html += """ <button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}"> <span class="glyphicon glyphicon-copy"></span> Copy table </button> """.format(tid=table_id) # Configure Columns Button if len(t_headers) > 1: html += """ <button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal"> <span class="glyphicon glyphicon-th"></span> Configure Columns </button> """.format(tid=table_id) # Sort By Highlight button html += """ <button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;"> <span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight </button> """.format(tid=table_id) # Scatter Plot Button if len(t_headers) > 1: html += """ <button type="button" class="mqc_table_makeScatter btn btn-default btn-sm" data-toggle="modal" data-target="#tableScatterModal" data-table="#{tid}"> <span class="glyphicon glyphicon glyphicon-stats"></span> Plot </button> """.format(tid=table_id) # "Showing x of y columns" text row_visibilities = [ all(t_rows_empty[s_name].values()) for s_name in t_rows_empty ] visible_rows = [x for x in row_visibilities if not x] # Visible rows t_showing_rows_txt = ( 'Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nvisrows}</sup>/<sub>{nrows}</sub> rows' .format(tid=table_id, nvisrows=len(visible_rows), nrows=len(t_rows))) # How many columns are visible? ncols_vis = (len(t_headers) + 1) - hidden_cols t_showing_cols_txt = "" if len(t_headers) > 1: t_showing_cols_txt = ' and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns'.format( tid=table_id, ncols_vis=ncols_vis, ncols=len(t_headers)) # Build table header text html += """ <small id="{tid}_numrows_text" class="mqc_table_numrows_text">{rows}{cols}.</small> """.format(tid=table_id, rows=t_showing_rows_txt, cols=t_showing_cols_txt) # Build the table itself collapse_class = "mqc-table-collapse" if len( t_rows) > 10 and config.collapse_tables else "" html += """ <div id="{tid}_container" class="mqc_table_container"> <div class="table-responsive mqc-table-responsive {cc}"> <table id="{tid}" class="table table-condensed mqc_table" data-title="{title}"> """.format(tid=table_id, title=table_title, cc=collapse_class) # Build the header row col1_header = dt.pconfig.get("col1_header", "Sample Name") html += '<thead><tr><th class="rowheader">{}</th>{}</tr></thead>'.format( col1_header, "".join(t_headers.values())) # Build the table body html += "<tbody>" t_row_keys = t_rows.keys() if dt.pconfig.get("sortRows") is not False: t_row_keys = sorted(t_row_keys) for s_name in t_row_keys: # Hide the row if all cells are empty or hidden row_hidden = ' style="display:none"' if all( t_rows_empty[s_name].values()) else "" html += "<tr{}>".format(row_hidden) # Sample name row header html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format( sn=s_name) for k in t_headers: html += t_rows[s_name].get(k, empty_cells[k]) html += "</tr>" html += "</tbody></table></div>" if len(t_rows) > 10 and config.collapse_tables: html += '<div class="mqc-table-expand"><span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span></div>' html += "</div>" # Build the bootstrap modal to customise columns and order if not config.simple_output: html += """ <!-- MultiQC Table Columns Modal --> <div class="modal fade" id="{tid}_configModal" tabindex="-1"> <div class="modal-dialog modal-lg"> <div class="modal-content"> <div class="modal-header"> <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button> <h4 class="modal-title">{title}: Columns</h4> </div> <div class="modal-body"> <p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p> <p> <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button> <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button> </p> <table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table" data-title="{title}"> <thead> <tr> <th class="sorthandle" style="text-align:center;">Sort</th> <th style="text-align:center;">Visible</th> <th>Group</th> <th>Column</th> <th>Description</th> <th>ID</th> <th>Scale</th> </tr> </thead> <tbody> {trows} </tbody> </table> </div> <div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div> </div> </div> </div>""".format(tid=table_id, title=table_title, trows="".join(t_modal_headers.values())) # Save the raw values to a file if requested if dt.pconfig.get("save_file") is True: fn = dt.pconfig.get("raw_data_fn", "multiqc_{}".format(table_id)) util_functions.write_data_file(dt.raw_vals, fn) report.saved_raw_data[fn] = dt.raw_vals return html
def matplotlib_linegraph(plotdata, pconfig=None): """ Plot a line graph with Matplot lib and return a HTML string. Either embeds a base64 encoded image within HTML or writes the plot and links to it. Should be called by plot_bargraph, which properly formats the input data. """ if pconfig is None: pconfig = {} # Plot group ID if pconfig.get("id") is None: pconfig["id"] = "mqc_mplplot_" + "".join(random.sample(letters, 10)) # Sanitise plot ID and check for duplicates pconfig["id"] = report.save_htmlid(pconfig["id"]) # Individual plot IDs pids = [] for k in range(len(plotdata)): try: name = pconfig["data_labels"][k]["name"] except: name = k + 1 pid = "mqc_{}_{}".format(pconfig["id"], name) pid = report.save_htmlid(pid, skiplint=True) pids.append(pid) html = ( '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> ' + "Flat image plot. Toolbox functions such as highlighting / hiding samples will not work " + '(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>' ) html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig["id"]) # Same defaults as HighCharts for consistency default_colors = [ "#7cb5ec", "#434348", "#90ed7d", "#f7a35c", "#8085e9", "#f15c80", "#e4d354", "#2b908f", "#f45b5b", "#91e8e1", ] # Buttons to cycle through different datasets if len(plotdata) > 1 and not config.simple_output: html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n' for k, p in enumerate(plotdata): pid = pids[k] active = "active" if k == 0 else "" try: name = pconfig["data_labels"][k]["name"] except: name = k + 1 html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format( a=active, pid=pid, n=name) html += "</div>\n\n" # Go through datasets creating plots for pidx, pdata in enumerate(plotdata): # Plot ID pid = pids[pidx] # Save plot data to file fdata = OrderedDict() lastcats = None sharedcats = True for d in pdata: fdata[d["name"]] = OrderedDict() for i, x in enumerate(d["data"]): if type(x) is list: fdata[d["name"]][str(x[0])] = x[1] # Check to see if all categories are the same if lastcats is None: lastcats = [x[0] for x in d["data"]] elif lastcats != [x[0] for x in d["data"]]: sharedcats = False else: try: fdata[d["name"]][pconfig["categories"][i]] = x except (KeyError, IndexError): fdata[d["name"]][str(i)] = x # Custom tsv output if the x axis varies if not sharedcats and config.data_format == "tsv": fout = "" for d in pdata: fout += "\t" + "\t".join([str(x[0]) for x in d["data"]]) fout += "\n{}\t".format(d["name"]) fout += "\t".join([str(x[1]) for x in d["data"]]) fout += "\n" with io.open(os.path.join(config.data_dir, "{}.txt".format(pid)), "w", encoding="utf-8") as f: print(fout.encode("utf-8", "ignore").decode("utf-8"), file=f) else: util_functions.write_data_file(fdata, pid) # Set up figure fig = plt.figure(figsize=(14, 6), frameon=False) axes = fig.add_subplot(111) # Go through data series for idx, d in enumerate(pdata): # Default colour index cidx = idx while cidx >= len(default_colors): cidx -= len(default_colors) # Line style linestyle = "solid" if d.get("dashStyle", None) == "Dash": linestyle = "dashed" # Reformat data (again) try: axes.plot( [x[0] for x in d["data"]], [x[1] for x in d["data"]], label=d["name"], color=d.get("color", default_colors[cidx]), linestyle=linestyle, linewidth=1, marker=None, ) except TypeError: # Categorical data on x axis axes.plot(d["data"], label=d["name"], color=d.get("color", default_colors[cidx]), linewidth=1, marker=None) # Tidy up axes axes.tick_params(labelsize=8, direction="out", left=False, right=False, top=False, bottom=False) axes.set_xlabel(pconfig.get("xlab", "")) axes.set_ylabel(pconfig.get("ylab", "")) # Dataset specific y label try: axes.set_ylabel(pconfig["data_labels"][pidx]["ylab"]) except: pass # Axis limits default_ylimits = axes.get_ylim() ymin = default_ylimits[0] if "ymin" in pconfig: ymin = pconfig["ymin"] elif "yFloor" in pconfig: ymin = max(pconfig["yFloor"], default_ylimits[0]) ymax = default_ylimits[1] if "ymax" in pconfig: ymax = pconfig["ymax"] elif "yCeiling" in pconfig: ymax = min(pconfig["yCeiling"], default_ylimits[1]) if (ymax - ymin) < pconfig.get("yMinRange", 0): ymax = ymin + pconfig["yMinRange"] axes.set_ylim((ymin, ymax)) # Dataset specific ymax try: axes.set_ylim((ymin, pconfig["data_labels"][pidx]["ymax"])) except: pass default_xlimits = axes.get_xlim() xmin = default_xlimits[0] if "xmin" in pconfig: xmin = pconfig["xmin"] elif "xFloor" in pconfig: xmin = max(pconfig["xFloor"], default_xlimits[0]) xmax = default_xlimits[1] if "xmax" in pconfig: xmax = pconfig["xmax"] elif "xCeiling" in pconfig: xmax = min(pconfig["xCeiling"], default_xlimits[1]) if (xmax - xmin) < pconfig.get("xMinRange", 0): xmax = xmin + pconfig["xMinRange"] axes.set_xlim((xmin, xmax)) # Plot title if "title" in pconfig: plt.text(0.5, 1.05, pconfig["title"], horizontalalignment="center", fontsize=16, transform=axes.transAxes) axes.grid(True, zorder=10, which="both", axis="y", linestyle="-", color="#dedede", linewidth=1) # X axis categories, if specified if "categories" in pconfig: axes.set_xticks([i for i, v in enumerate(pconfig["categories"])]) axes.set_xticklabels(pconfig["categories"]) # Axis lines xlim = axes.get_xlim() axes.plot([xlim[0], xlim[1]], [0, 0], linestyle="-", color="#dedede", linewidth=2) axes.set_axisbelow(True) axes.spines["right"].set_visible(False) axes.spines["top"].set_visible(False) axes.spines["bottom"].set_visible(False) axes.spines["left"].set_visible(False) # Background colours, if specified if "yPlotBands" in pconfig: xlim = axes.get_xlim() for pb in pconfig["yPlotBands"]: axes.barh( pb["from"], xlim[1], height=pb["to"] - pb["from"], left=xlim[0], color=pb["color"], linewidth=0, zorder=0, align="edge", ) if "xPlotBands" in pconfig: ylim = axes.get_ylim() for pb in pconfig["xPlotBands"]: axes.bar( pb["from"], ylim[1], width=pb["to"] - pb["from"], bottom=ylim[0], color=pb["color"], linewidth=0, zorder=0, align="edge", ) # Tight layout - makes sure that legend fits in and stuff if len(pdata) <= 15: axes.legend( loc="lower center", bbox_to_anchor=(0, -0.22, 1, 0.102), ncol=5, mode="expand", fontsize=8, frameon=False, ) plt.tight_layout(rect=[0, 0.08, 1, 0.92]) else: plt.tight_layout(rect=[0, 0, 1, 0.92]) # Should this plot be hidden on report load? hidediv = "" if pidx > 0: hidediv = ' style="display:none;"' # Save the plot to the data directory if export is requests if config.export_plots: for fformat in config.export_plot_formats: # Make the directory if it doesn't already exist plot_dir = os.path.join(config.plots_dir, fformat) if not os.path.exists(plot_dir): os.makedirs(plot_dir) # Save the plot plot_fn = os.path.join(plot_dir, "{}.{}".format(pid, fformat)) fig.savefig(plot_fn, format=fformat, bbox_inches="tight") # Output the figure to a base64 encoded string if getattr(get_template_mod(), "base64_plots", True) is True: img_buffer = io.BytesIO() fig.savefig(img_buffer, format="png", bbox_inches="tight") b64_img = base64.b64encode(img_buffer.getvalue()).decode("utf8") img_buffer.close() html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format( pid, hidediv, b64_img) # Save to a file and link <img> else: plot_relpath = os.path.join(config.plots_dir_name, "png", "{}.png".format(pid)) html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format( pid, hidediv, plot_relpath) plt.close(fig) # Close wrapping div html += "</div>" report.num_mpl_plots += 1 return html
def matplotlib_bargraph (plotdata, plotsamples, pconfig=None): """ Plot a bargraph with Matplot lib and return a HTML string. Either embeds a base64 encoded image within HTML or writes the plot and links to it. Should be called by plot_bargraph, which properly formats the input data. """ if pconfig is None: pconfig = {} # Plot group ID if pconfig.get('id') is None: pconfig['id'] = 'mqc_mplplot_'+''.join(random.sample(letters, 10)) # Sanitise plot ID and check for duplicates pconfig['id'] = report.save_htmlid(pconfig['id']) # Individual plot IDs pids = [] for k in range(len(plotdata)): try: name = pconfig['data_labels'][k] except: name = k+1 pid = 'mqc_{}_{}'.format(pconfig['id'], name) pid = report.save_htmlid(pid, skiplint=True) pids.append(pid) html = '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> ' + \ 'Flat image plot. Toolbox functions such as highlighting / hiding samples will not work ' + \ '(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>' html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig['id']) # Same defaults as HighCharts for consistency default_colors = ['#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1'] # Counts / Percentages Switch if pconfig.get('cpswitch') is not False and not config.simple_output: if pconfig.get('cpswitch_c_active', True) is True: c_active = 'active' p_active = '' else: c_active = '' p_active = 'active' pconfig['stacking'] = 'percent' c_label = pconfig.get('cpswitch_counts_label', 'Counts') p_label = pconfig.get('cpswitch_percent_label', 'Percentages') html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_setcountspcnt"> \n\ <button class="btn btn-default btn-sm {c_a} counts">{c_l}</button> \n\ <button class="btn btn-default btn-sm {p_a} pcnt">{p_l}</button> \n\ </div> '.format(c_a=c_active, p_a=p_active, c_l=c_label, p_l=p_label) if len(plotdata) > 1: html += ' ' # Buttons to cycle through different datasets if len(plotdata) > 1 and not config.simple_output: html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n' for k, p in enumerate(plotdata): pid = pids[k] active = 'active' if k == 0 else '' try: name = pconfig['data_labels'][k] except: name = k+1 html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format(a=active, pid=pid, n=name) html += '</div>\n\n' # Go through datasets creating plots for pidx, pdata in enumerate(plotdata): # Save plot data to file fdata = {} for d in pdata: for didx, dval in enumerate(d['data']): s_name = plotsamples[pidx][didx] if s_name not in fdata: fdata[s_name] = dict() fdata[s_name][d['name']] = dval util_functions.write_data_file(fdata, pids[pidx]) # Plot percentage as well as counts plot_pcts = [False] if pconfig.get('cpswitch') is not False: plot_pcts = [False, True] # Switch out NaN for 0s so that MatPlotLib doesn't ignore stuff for idx, d in enumerate(pdata): pdata[idx]['data'] = [x if not math.isnan(x) else 0 for x in d['data'] ] for plot_pct in plot_pcts: # Plot ID pid = pids[pidx] hide_plot = False if plot_pct is True: pid = '{}_pc'.format(pid) if pconfig.get('cpswitch_c_active', True) is True: hide_plot = True else: if pconfig.get('cpswitch_c_active', True) is not True: hide_plot = True # Set up figure plt_height = len(plotsamples[pidx]) / 2.3 plt_height = max(6, plt_height) # At least 6" tall plt_height = min(30, plt_height) # Cap at 30" tall bar_width = 0.8 fig = plt.figure(figsize=(14, plt_height), frameon=False) axes = fig.add_subplot(111) y_ind = range(len(plotsamples[pidx])) # Count totals for each sample if plot_pct is True: s_totals = [0 for _ in pdata[0]['data']] for series_idx, d in enumerate(pdata): for sample_idx, v in enumerate(d['data']): s_totals[sample_idx] += v # Plot bars dlabels = [] for idx, d in enumerate(pdata): # Plot percentages values = d['data'] if len(values) < len(y_ind): values.extend([0] * (len(y_ind) - len(values))) if plot_pct is True: for (key,var) in enumerate(values): s_total = s_totals[key] if s_total == 0: values[key] = 0 else: values[key] = (float(var+0.0)/float(s_total))*100 # Get offset for stacked bars if idx == 0: prevdata = [0] * len(plotsamples[pidx]) else: for i, p in enumerate(prevdata): prevdata[i] += pdata[idx-1]['data'][i] # Default colour index cidx = idx while cidx >= len(default_colors): cidx -= len(default_colors) # Save the name of this series dlabels.append(d['name']) # Add the series of bars to the plot axes.barh( y_ind, values, bar_width, left = prevdata, color = d.get('color', default_colors[cidx]), align = 'center', linewidth = pconfig.get('borderWidth', 0) ) # Tidy up axes axes.tick_params(labelsize=8, direction='out', left=False, right=False, top=False, bottom=False) axes.set_xlabel(pconfig.get('ylab', '')) # I know, I should fix the fact that the config is switched axes.set_ylabel(pconfig.get('xlab', '')) axes.set_yticks(y_ind) # Specify where to put the labels axes.set_yticklabels(plotsamples[pidx]) # Set y axis sample name labels axes.set_ylim((-0.5, len(y_ind)-0.5)) # Reduce padding around plot area if plot_pct is True: axes.set_xlim((0, 100)) # Add percent symbols vals = axes.get_xticks() axes.set_xticklabels(['{:.0f}%'.format(x) for x in vals]) else: default_xlimits = axes.get_xlim() axes.set_xlim((pconfig.get('ymin', default_xlimits[0]),pconfig.get('ymax', default_xlimits[1]))) if 'title' in pconfig: top_gap = 1 + (0.5 / plt_height) plt.text(0.5, top_gap, pconfig['title'], horizontalalignment='center', fontsize=16, transform=axes.transAxes) axes.grid(True, zorder=0, which='both', axis='x', linestyle='-', color='#dedede', linewidth=1) axes.set_axisbelow(True) axes.spines['right'].set_visible(False) axes.spines['top'].set_visible(False) axes.spines['bottom'].set_visible(False) axes.spines['left'].set_visible(False) plt.gca().invert_yaxis() # y axis is reverse sorted otherwise # Hide some labels if we have a lot of samples show_nth = max(1, math.ceil(len(pdata[0]['data'])/150)) for idx, label in enumerate(axes.get_yticklabels()): if idx % show_nth != 0: label.set_visible(False) # Legend bottom_gap = -1 * (1 - ((plt_height - 1.5) / plt_height)) lgd = axes.legend(dlabels, loc='lower center', bbox_to_anchor=(0, bottom_gap, 1, .102), ncol=5, mode='expand', fontsize=8, frameon=False) # Should this plot be hidden on report load? hidediv = '' if pidx > 0 or hide_plot: hidediv = ' style="display:none;"' # Save the plot to the data directory if export is requested if config.export_plots: for fformat in config.export_plot_formats: # Make the directory if it doesn't already exist plot_dir = os.path.join(config.plots_dir, fformat) if not os.path.exists(plot_dir): os.makedirs(plot_dir) # Save the plot plot_fn = os.path.join(plot_dir, '{}.{}'.format(pid, fformat)) fig.savefig(plot_fn, format=fformat, bbox_extra_artists=(lgd,), bbox_inches='tight') # Output the figure to a base64 encoded string if getattr(get_template_mod(), 'base64_plots', True) is True: img_buffer = io.BytesIO() fig.savefig(img_buffer, format='png', bbox_inches='tight') b64_img = base64.b64encode(img_buffer.getvalue()).decode('utf8') img_buffer.close() html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format(pid, hidediv, b64_img) # Link to the saved image else: plot_relpath = os.path.join(config.plots_dir_name, 'png', '{}.png'.format(pid)) html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(pid, hidediv, plot_relpath) plt.close(fig) # Close wrapping div html += '</div>' report.num_mpl_plots += 1 return html
def make_table(dt, hide_bar=None): """ Build the HTML needed for a MultiQC table. :param data: MultiQC datatable object """ table_id = dt.pconfig.get( 'id', 'table_{}'.format(''.join(random.sample(letters, 4)))) table_id = report.save_htmlid(table_id) t_headers = OrderedDict() t_modal_headers = OrderedDict() t_rows = OrderedDict() t_rows_empty = OrderedDict() dt.raw_vals = defaultdict(lambda: dict()) empty_cells = dict() hidden_cols = 1 table_title = dt.pconfig.get('table_title') if table_title is None: table_title = table_id.replace("_", " ").title() for idx, k, header in dt.get_headers_in_order(): rid = header['rid'] # Build the table header cell shared_key = '' if header.get('shared_key', None) is not None: shared_key = ' data-shared-key={}'.format(header['shared_key']) hide = '' muted = '' checked = ' checked="checked"' if header.get('hidden', False) is True: hide = 'hidden' muted = ' text-muted' checked = '' hidden_cols += 1 data_attr = 'data-dmax="{}" data-dmin="{}" data-namespace="{}" {}' \ .format(header['dmax'], header['dmin'], header['namespace'], shared_key) if header.get('namespace'): cell_contents = '<span class="mqc_table_tooltip" title="{}: {}">{}</span>' \ .format(header['namespace'], header['description'], header['title']) else: cell_contents = '<span class="mqc_table_tooltip" title="{} {}">{}</span>' \ .format(header['namespace'], header['description'], header['title']) t_headers[rid] = '<th id="header_{rid}" class="{rid} {h}" {da}>{c}</th>' \ .format(rid=rid, h=hide, da=data_attr, c=cell_contents) empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format( rid=rid, h=hide) # Build the modal table row t_modal_headers[rid] = """ <tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);"> <td class="sorthandle ui-sortable-handle">||</span></td> <td style="text-align:center;"> <input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}"> </td> <td>{name}</td> <td>{title}</td> <td>{desc}</td> <td>{col_id}</td> <td>{sk}</td> </tr>""".format(rid=rid, muted=muted, checked=checked, tid=table_id, col=header['colour'], name=header['namespace'], title=header['title'], desc=header['description'], col_id='<code>{}</code>'.format(k), sk=header.get('shared_key', '')) # Make a colour scale if header['scale'] == False: c_scale = None else: c_scale = mqc_colour.mqc_colour_scale(header['scale'], header['dmin'], header['dmax']) # Add the data table cells for (s_name, samp) in dt.data[idx].items(): if k in samp: val = samp[k] kname = '{}_{}'.format(header['namespace'], rid) dt.raw_vals[s_name][kname] = val # if "is_int" in header.keys(): # val = int(val) # print(val) if 'modify' in header and callable(header['modify']): val = header['modify'](val) try: dmin = header['dmin'] dmax = header['dmax'] percentage = ((float(val) - dmin) / (dmax - dmin)) * 100 percentage = min(percentage, 100) percentage = max(percentage, 0) except (ZeroDivisionError, ValueError): percentage = 0 try: if not header.get("is_int"): valstring = str(header['format'].format(val)) else: valstring = str(int(val)) except ValueError: try: valstring = str(header['format'].format(float(val))) except ValueError: valstring = str(val) except: valstring = str(val) # This is horrible, but Python locale settings are worse if config.thousandsSep_format is None: config.thousandsSep_format = '<span class="mqc_thousandSep"></span>' if config.decimalPoint_format is None: config.decimalPoint_format = '.' valstring = valstring.replace('.', 'DECIMAL').replace( ',', 'THOUSAND') valstring = valstring.replace( 'DECIMAL', config.decimalPoint_format).replace( 'THOUSAND', config.thousandsSep_format) # Percentage suffixes etc if header.get('suffix') == "show_perc": suff_dict = header.get("suffix_dict") valstring += str(suff_dict.get(s_name)) else: valstring += header.get('suffix', '') # Conditional formatting cmatches = { cfck: False for cfc in config.table_cond_formatting_colours for cfck in cfc } # Find general rules followed by column-specific rules for cfk in ['all_columns', rid]: if cfk in config.table_cond_formatting_rules: # Loop through match types for ftype in cmatches.keys(): # Loop through array of comparison types for cmp in config.table_cond_formatting_rules[ cfk].get(ftype, []): try: # Each comparison should be a dict with single key: val if 's_eq' in cmp and str( cmp['s_eq']).lower() == str( val).lower(): cmatches[ftype] = True if 's_contains' in cmp and str( cmp['s_contains']).lower() in str( val).lower(): cmatches[ftype] = True if 's_ne' in cmp and str( cmp['s_ne']).lower() != str( val).lower(): cmatches[ftype] = True if 'eq' in cmp and float( cmp['eq']) == float(val): cmatches[ftype] = True if 'ne' in cmp and float( cmp['ne']) != float(val): cmatches[ftype] = True if 'gt' in cmp and float( cmp['gt']) < float(val): cmatches[ftype] = True if 'lt' in cmp and float( cmp['lt']) > float(val): cmatches[ftype] = True except: logger.warn( "Not able to apply table conditional formatting to '{}' ({})" .format(val, cmp)) # Apply HTML in order of config keys bgcol = None for cfc in config.table_cond_formatting_colours: for cfck in cfc: # should always be one, but you never know if cmatches[cfck]: bgcol = cfc[cfck] if bgcol is not None: valstring = '<span class="badge" style="background-color:{}">{}</span>'.format( bgcol, valstring) # Build HTML if not header['scale']: if s_name not in t_rows: t_rows[s_name] = dict() t_rows[s_name][ rid] = '<td class="{rid} {h}">{v}</td>'.format( rid=rid, h=hide, v=valstring) # coloring with quartiles elif header['scale'] == "quart": col_dict = header['col_dict'] bar_dict = header.get('bar_dict') col = ' background-color:{};'.format(col_dict.get(s_name)) if bar_dict: bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format( bar_dict.get(s_name) + 3, col) else: bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format( percentage, col) # bar percentage here val_html = '<span class="val">{}</span>'.format(valstring) wrapper_html = '<div class="wrapper">{}{}</div>'.format( bar_html, val_html) if s_name not in t_rows: t_rows[s_name] = dict() t_rows[s_name][ rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format( rid=rid, h=hide, c=wrapper_html) else: if c_scale is not None: col = ' background-color:{};'.format( c_scale.get_colour(val)) else: col = '' bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format( percentage, col) val_html = '<span class="val">{}</span>'.format(valstring) wrapper_html = '<div class="wrapper">{}{}</div>'.format( bar_html, val_html) if s_name not in t_rows: t_rows[s_name] = dict() t_rows[s_name][ rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format( rid=rid, h=hide, c=wrapper_html) # Is this cell hidden or empty? if s_name not in t_rows_empty: t_rows_empty[s_name] = dict() t_rows_empty[s_name][rid] = header.get( 'hidden', False) or str(val).strip() == '' # Remove header if we don't have any filled cells for it if sum([len(rows) for rows in t_rows.values()]) == 0: t_headers.pop(rid, None) t_modal_headers.pop(rid, None) logger.debug( 'Removing header {} from general stats table, as no data'. format(k)) # # Put everything together # # Buttons above the table html = '' if not config.simple_output: # Copy Table Button html += """<div class="row"> <div class="col-sm-2"> <button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}"> <span class="glyphicon glyphicon-copy"></span> Copy table </button> </div> """.format(tid=table_id) # <div class="progress-bar progress-bar-warning progress-bar-striped" style="width: 20%"> # <span class="sr-only">20% Complete (warning)</span> # </div> # <div class="progress-bar progress-bar-danger" style="width: 10%"> # <span class="sr-only">10% Complete (danger)</span> # </div> if not hide_bar: html += """ <div class="col-sm-2"> <div id="quartiles_bar"> <div class="progress"> <div class="progress-bar progress-bar-q1" style="width: 25%"> <span class="sr-only">35% Complete (success)</span> Q1 </div> <div class="progress-bar progress-bar-q2" style="width: 25%"> <span class="sr-only">35% Complete (success)</span> Q2 </div> <div class="progress-bar progress-bar-q3" style="width: 25%"> <span class="sr-only">35% Complete (success)</span> Q3 </div> <div class="progress-bar progress-bar-q4" style="width: 25%"> <span class="sr-only">35% Complete (success)</span> Q4 </div> </div> </div> </div> </div> """ else: html += """ </div> """ # Configure Columns Button # if len(t_headers) > 1: # html += """ # <button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal"> # <span class="glyphicon glyphicon-th"></span> Configure Columns # </button> # """.format(tid=table_id) # Sort By Highlight button # html += """ # <button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;"> # <span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight # </button> # """.format(tid=table_id) # Scatter Plot Button # if len(t_headers) > 1: # html += """ # <button type="button" class="mqc_table_makeScatter btn btn-default btn-sm" data-toggle="modal" data-target="#tableScatterModal" data-table="#{tid}"> # <span class="glyphicon glyphicon glyphicon-stats"></span> Plot # </button> # """.format(tid=table_id) # "Showing x of y columns" text row_visibilities = [ all(t_rows_empty[s_name].values()) for s_name in t_rows_empty ] visible_rows = [x for x in row_visibilities if not x] # html += """ # <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nvisrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small> # """.format(tid=table_id, nvisrows=len(visible_rows), nrows=len(t_rows), ncols_vis = (len(t_headers)+1)-hidden_cols, ncols=len(t_headers)) # Add text # html += """ # <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nvisrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small> # """.format(tid=table_id, nvisrows=len(visible_rows), nrows=len(t_rows), # ncols_vis=(len(t_headers) + 1) - hidden_cols, ncols=len(t_headers)) # Build the table itself collapse_class = 'mqc-table-collapse' if len( t_rows) > 10 and config.collapse_tables else '' html += """ <div id="{tid}_container" class="mqc_table_container"> <div class="table-responsive mqc-table-responsive {cc}"> <table id="{tid}" class="table table-condensed mqc_table" data-title="{title}"> """.format(tid=table_id, title=table_title, cc=collapse_class) # Build the header row col1_header = dt.pconfig.get('col1_header', 'Sample Name') html += '<thead><tr><th class="rowheader">{}</th>{}</tr></thead>'.format( col1_header, ''.join(t_headers.values())) # Build the table body html += '<tbody>' t_row_keys = t_rows.keys() if dt.pconfig.get('sortRows') is not False: t_row_keys = sorted(t_row_keys) for s_name in t_row_keys: # Hide the row if all cells are empty or hidden row_hidden = ' style="display:none"' if all( t_rows_empty[s_name].values()) else '' html += '<tr{}>'.format(row_hidden) # Sample name row header html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format( sn=s_name) for k in t_headers: html += t_rows[s_name].get(k, empty_cells[k]) html += '</tr>' html += '</tbody></table></div>' if len(t_rows) > 10 and config.collapse_tables: html += '<div class="mqc-table-expand"><span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span></div>' html += '</div>' # Build the bootstrap modal to customise columns and order # if not config.simple_output: # html += """ # <!-- MultiQC Table Columns Modal --> # <div class="modal fade" id="{tid}_configModal" tabindex="-1"> # <div class="modal-dialog modal-lg"> # <div class="modal-content"> # <div class="modal-header"> # <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button> # <h4 class="modal-title">{title}: Columns</h4> # </div> # <div class="modal-body"> # <p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p> # <p> # <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button> # <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button> # </p> # <table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table" data-title="{title}"> # <thead> # <tr> # <th class="sorthandle" style="text-align:center;">Sort</th> # <th style="text-align:center;">Visible</th> # <th>Group</th> # <th>Column</th> # <th>Description</th> # <th>ID</th> # <th>Scale</th> # </tr> # </thead> # <tbody> # {trows} # </tbody> # </table> # </div> # <div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div> # </div> </div> </div>""".format( tid=table_id, title=table_title, trows=''.join(t_modal_headers.values()) ) # Save the raw values to a file if requested if dt.pconfig.get('save_file') is True: fn = dt.pconfig.get('raw_data_fn', 'multiqc_{}'.format(table_id)) util_functions.write_data_file(dt.raw_vals, fn) report.saved_raw_data[fn] = dt.raw_vals return html
def matplotlib_bargraph(plotdata, plotsamples, pconfig=None): """ Plot a bargraph with Matplot lib and return a HTML string. Either embeds a base64 encoded image within HTML or writes the plot and links to it. Should be called by plot_bargraph, which properly formats the input data. """ if pconfig is None: pconfig = {} # Plot group ID if pconfig.get("id") is None: pconfig["id"] = "mqc_mplplot_" + "".join(random.sample(letters, 10)) # Sanitise plot ID and check for duplicates pconfig["id"] = report.save_htmlid(pconfig["id"]) # Individual plot IDs pids = [] for k in range(len(plotdata)): try: name = pconfig["data_labels"][k] except: name = k + 1 pid = "mqc_{}_{}".format(pconfig["id"], name) pid = report.save_htmlid(pid, skiplint=True) pids.append(pid) html = ( '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> ' + "Flat image plot. Toolbox functions such as highlighting / hiding samples will not work " + '(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>' ) html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig["id"]) # Same defaults as HighCharts for consistency default_colors = [ "#7cb5ec", "#434348", "#90ed7d", "#f7a35c", "#8085e9", "#f15c80", "#e4d354", "#2b908f", "#f45b5b", "#91e8e1", ] # Counts / Percentages Switch if pconfig.get("cpswitch") is not False and not config.simple_output: if pconfig.get("cpswitch_c_active", True) is True: c_active = "active" p_active = "" else: c_active = "" p_active = "active" pconfig["stacking"] = "percent" c_label = pconfig.get("cpswitch_counts_label", "Counts") p_label = pconfig.get("cpswitch_percent_label", "Percentages") html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_setcountspcnt"> \n\ <button class="btn btn-default btn-sm {c_a} counts">{c_l}</button> \n\ <button class="btn btn-default btn-sm {p_a} pcnt">{p_l}</button> \n\ </div> '.format( c_a=c_active, p_a=p_active, c_l=c_label, p_l=p_label ) if len(plotdata) > 1: html += " " # Buttons to cycle through different datasets if len(plotdata) > 1 and not config.simple_output: html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n' for k, p in enumerate(plotdata): pid = pids[k] active = "active" if k == 0 else "" try: name = pconfig["data_labels"][k] except: name = k + 1 html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format( a=active, pid=pid, n=name ) html += "</div>\n\n" # Go through datasets creating plots for pidx, pdata in enumerate(plotdata): # Save plot data to file fdata = {} for d in pdata: for didx, dval in enumerate(d["data"]): s_name = plotsamples[pidx][didx] if s_name not in fdata: fdata[s_name] = dict() fdata[s_name][d["name"]] = dval if pconfig.get("save_data_file", True): util_functions.write_data_file(fdata, pids[pidx]) # Plot percentage as well as counts plot_pcts = [False] if pconfig.get("cpswitch") is not False: plot_pcts = [False, True] # Switch out NaN for 0s so that MatPlotLib doesn't ignore stuff for idx, d in enumerate(pdata): pdata[idx]["data"] = [x if not math.isnan(x) else 0 for x in d["data"]] for plot_pct in plot_pcts: # Plot ID pid = pids[pidx] hide_plot = False if plot_pct is True: pid = "{}_pc".format(pid) if pconfig.get("cpswitch_c_active", True) is True: hide_plot = True else: if pconfig.get("cpswitch_c_active", True) is not True: hide_plot = True # Set up figure # Height has a default, then adjusted by the number of samples plt_height = len(plotsamples[pidx]) / 2.3 # Default in inches, empirically determined plt_height = max(6, plt_height) # At least 6" tall plt_height = min(30, plt_height) # Cap at 30" tall # Use fixed height if pconfig['height'] is set (convert pixels -> inches) if "height" in pconfig: # Default interactive height in pixels = 512 # Not perfect replication, but good enough plt_height = 6 * (pconfig["height"] / 512) bar_width = 0.8 fig = plt.figure(figsize=(14, plt_height), frameon=False) axes = fig.add_subplot(111) y_ind = range(len(plotsamples[pidx])) # Count totals for each sample if plot_pct is True: s_totals = [0 for _ in pdata[0]["data"]] for series_idx, d in enumerate(pdata): for sample_idx, v in enumerate(d["data"]): s_totals[sample_idx] += v # Plot bars dlabels = [] prev_values = None for idx, d in enumerate(pdata): # Plot percentages values = [x for x in d["data"]] if len(values) < len(y_ind): values.extend([0] * (len(y_ind) - len(values))) if plot_pct is True: for (key, var) in enumerate(values): s_total = s_totals[key] if s_total == 0: values[key] = 0 else: values[key] = (float(var + 0.0) / float(s_total)) * 100 # Get offset for stacked bars if idx == 0: prevdata = [0] * len(plotsamples[pidx]) else: for i, p in enumerate(prevdata): prevdata[i] += prev_values[i] # Default colour index cidx = idx while cidx >= len(default_colors): cidx -= len(default_colors) # Save the name of this series dlabels.append(d["name"]) # Add the series of bars to the plot axes.barh( y_ind, values, bar_width, left=prevdata, color=d.get("color", default_colors[cidx]), align="center", linewidth=pconfig.get("borderWidth", 0), ) prev_values = values # Tidy up axes axes.tick_params( labelsize=pconfig.get("labelSize", 8), direction="out", left=False, right=False, top=False, bottom=False ) axes.set_xlabel(pconfig.get("ylab", "")) # I know, I should fix the fact that the config is switched axes.set_ylabel(pconfig.get("xlab", "")) axes.set_yticks(y_ind) # Specify where to put the labels axes.set_yticklabels(plotsamples[pidx]) # Set y axis sample name labels axes.set_ylim((-0.5, len(y_ind) - 0.5)) # Reduce padding around plot area if plot_pct is True: axes.set_xlim((0, 100)) # Add percent symbols vals = axes.get_xticks() axes.set_xticks(axes.get_xticks()) axes.set_xticklabels(["{:.0f}%".format(x) for x in vals]) else: default_xlimits = axes.get_xlim() axes.set_xlim((pconfig.get("ymin", default_xlimits[0]), pconfig.get("ymax", default_xlimits[1]))) if "title" in pconfig: top_gap = 1 + (0.5 / plt_height) plt.text( 0.5, top_gap, pconfig["title"], horizontalalignment="center", fontsize=16, transform=axes.transAxes ) axes.grid(True, zorder=0, which="both", axis="x", linestyle="-", color="#dedede", linewidth=1) axes.set_axisbelow(True) axes.spines["right"].set_visible(False) axes.spines["top"].set_visible(False) axes.spines["bottom"].set_visible(False) axes.spines["left"].set_visible(False) plt.gca().invert_yaxis() # y axis is reverse sorted otherwise # Hide some labels if we have a lot of samples show_nth = max(1, math.ceil(len(pdata[0]["data"]) / 150)) for idx, label in enumerate(axes.get_yticklabels()): if idx % show_nth != 0: label.set_visible(False) # Legend bottom_gap = -1 * (1 - ((plt_height - 1.5) / plt_height)) lgd = axes.legend( dlabels, loc="lower center", bbox_to_anchor=(0, bottom_gap, 1, 0.102), ncol=5, mode="expand", fontsize=pconfig.get("labelSize", 8), frameon=False, ) # Should this plot be hidden on report load? hidediv = "" if pidx > 0 or hide_plot: hidediv = ' style="display:none;"' # Save the plot to the data directory if export is requested if config.export_plots: for fformat in config.export_plot_formats: # Make the directory if it doesn't already exist plot_dir = os.path.join(config.plots_dir, fformat) if not os.path.exists(plot_dir): os.makedirs(plot_dir) # Save the plot plot_fn = os.path.join(plot_dir, "{}.{}".format(pid, fformat)) fig.savefig(plot_fn, format=fformat, bbox_extra_artists=(lgd,), bbox_inches="tight") # Output the figure to a base64 encoded string if getattr(get_template_mod(), "base64_plots", True) is True: img_buffer = io.BytesIO() fig.savefig(img_buffer, format="png", bbox_inches="tight") b64_img = base64.b64encode(img_buffer.getvalue()).decode("utf8") img_buffer.close() html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format( pid, hidediv, b64_img ) # Link to the saved image else: plot_relpath = os.path.join(config.plots_dir_name, "png", "{}.png".format(pid)) html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(pid, hidediv, plot_relpath) plt.close(fig) # Close wrapping div html += "</div>" return html
def multiqc(analysis_dir, dirs, dirs_depth, no_clean_sname, title, report_comment, template, module_tag, module, exclude, outdir, ignore, ignore_samples, sample_names, file_list, filename, make_data_dir, no_data_dir, data_format, zip_data_dir, force, ignore_symlinks, export_plots, plots_flat, plots_interactive, lint, make_pdf, no_megaqc_upload, config_file, cl_config, verbose, quiet, **kwargs): """MultiQC aggregates results from bioinformatics analyses across many samples into a single report. It searches a given directory for analysis logs and compiles a HTML report. It's a general use tool, perfect for summarising the output from numerous bioinformatics tools. To run, supply with one or more directory to scan for analysis results. To run here, use 'multiqc .' See http://multiqc.info for more details. Author: Phil Ewels (http://phil.ewels.co.uk) """ # Set up logging level loglevel = log.LEVELS.get(min(verbose, 1), "INFO") if quiet: loglevel = 'WARNING' log.init_log(logger, loglevel=loglevel) # Load config files plugin_hooks.mqc_trigger('before_config') config.mqc_load_userconfig(config_file) plugin_hooks.mqc_trigger('config_loaded') # Command-line config YAML if len(cl_config) > 0: config.mqc_cl_config(cl_config) # Log the command used to launch MultiQC report.multiqc_command = " ".join(sys.argv) logger.debug("Command used: {}".format(report.multiqc_command)) # Check that we're running the latest version of MultiQC if config.no_version_check is not True: try: response = urlopen('http://multiqc.info/version.php?v={}'.format( config.short_version), timeout=5) remote_version = response.read().decode('utf-8').strip() if version.StrictVersion(re.sub( '[^0-9\.]', '', remote_version)) > version.StrictVersion( re.sub('[^0-9\.]', '', config.short_version)): logger.warn( 'MultiQC Version {} now available!'.format(remote_version)) else: logger.debug( 'Latest MultiQC version is {}'.format(remote_version)) except Exception as e: logger.debug( 'Could not connect to multiqc.info for version check: {}'. format(e)) # Set up key variables (overwrite config vars from command line) if template is not None: config.template = template if title is not None: config.title = title if report_comment is not None: config.report_comment = report_comment if dirs is True: config.prepend_dirs = dirs if dirs_depth is not None: config.prepend_dirs = True config.prepend_dirs_depth = dirs_depth config.analysis_dir = analysis_dir if outdir is not None: config.output_dir = outdir if no_clean_sname: config.fn_clean_sample_names = False logger.info("Not cleaning sample names") if make_data_dir: config.make_data_dir = True if no_data_dir: config.make_data_dir = False if force: config.force = True if ignore_symlinks: config.ignore_symlinks = True if zip_data_dir: config.zip_data_dir = True if data_format is not None: config.data_format = data_format if export_plots: config.export_plots = True if plots_flat: config.plots_force_flat = True if plots_interactive: config.plots_force_interactive = True if lint: config.lint = True lint_helpers.run_tests() if make_pdf: config.template = 'simple' if no_megaqc_upload: config.megaqc_upload = False else: config.megaqc_upload = True if sample_names: config.load_sample_names(sample_names) if module_tag is not None: config.module_tag = module_tag config.kwargs = kwargs # Plugin command line options plugin_hooks.mqc_trigger('execution_start') logger.info("This is MultiQC v{}".format(__version__)) logger.debug("Command : {}".format(' '.join(sys.argv))) logger.debug("Working dir : {}".format(os.getcwd())) if make_pdf: logger.info('--pdf specified. Using non-interactive HTML template.') logger.info("Template : {}".format(config.template)) if lint: logger.info('--lint specified. Being strict with validation.') # Add files if --file-list option is given if file_list: if len(analysis_dir) > 1: raise ValueError( "If --file-list is giving, analysis_dir should have only one plain text file." ) config.analysis_dir = [] with (open(analysis_dir[0])) as in_handle: for line in in_handle: if os.path.exists(line.strip()): path = os.path.abspath(line.strip()) config.analysis_dir.append(path) if len(config.analysis_dir) == 0: logger.error( "No files or directories were added from {} using --file-list option." .format(analysis_dir[0])) logger.error( "Please, check that {} contains correct paths.".format( analysis_dir[0])) raise ValueError("Any files or directories to be searched.") if len(ignore) > 0: logger.debug( "Ignoring files, directories and paths that match: {}".format( ", ".join(ignore))) config.fn_ignore_files.extend(ignore) config.fn_ignore_dirs.extend(ignore) config.fn_ignore_paths.extend(ignore) if len(ignore_samples) > 0: logger.debug("Ignoring sample names that match: {}".format( ", ".join(ignore_samples))) config.sample_names_ignore.extend(ignore_samples) if filename == 'stdout': config.output_fn = sys.stdout logger.info("Printing report to stdout") else: if title is not None and filename is None: filename = re.sub('[^\w\.-]', '', re.sub('[-\s]+', '-', title)).strip() filename += '_multiqc_report' if filename is not None: if filename.endswith('.html'): filename = filename[:-5] config.output_fn_name = filename config.data_dir_name = '{}_data'.format(filename) if not config.output_fn_name.endswith('.html'): config.output_fn_name = '{}.html'.format(config.output_fn_name) # Print some status updates if config.title is not None: logger.info("Report title: {}".format(config.title)) if dirs: logger.info("Prepending directory to sample names") for d in config.analysis_dir: logger.info("Searching '{}'".format(d)) # Prep module configs config.top_modules = [ m if type(m) is dict else { m: {} } for m in config.top_modules ] config.module_order = [ m if type(m) is dict else { m: {} } for m in config.module_order ] mod_keys = [list(m.keys())[0] for m in config.module_order] # Lint the module configs if config.lint: for m in config.avail_modules.keys(): if m not in mod_keys: errmsg = "LINT: Module '{}' not found in config.module_order".format( m) logger.error(errmsg) report.lint_errors.append(errmsg) else: for mo in config.module_order: if m != 'custom_content' and m in mo.keys( ) and 'module_tag' not in mo[m]: errmsg = "LINT: Module '{}' in config.module_order did not have 'module_tag' config".format( m) logger.error(errmsg) report.lint_errors.append(errmsg) # Get the avaiable tags to decide which modules to run. modules_from_tags = set() if config.module_tag is not None: tags = config.module_tag for m in config.module_order: module_name = list(m.keys())[0] # only one name in each dict for tag in tags: for t in m[module_name].get('module_tag', []): if tag.lower() == t.lower(): modules_from_tags.add(module_name) # Get the list of modules we want to run, in the order that we want them run_modules = [ m for m in config.top_modules if list(m.keys())[0] in config.avail_modules.keys() ] run_modules.extend([{ m: {} } for m in config.avail_modules.keys() if m not in mod_keys and m not in run_modules]) run_modules.extend([ m for m in config.module_order if list(m.keys())[0] in config.avail_modules.keys() and list(m.keys()) [0] not in [list(rm.keys())[0] for rm in run_modules] ]) if module: run_modules = [m for m in run_modules if list(m.keys())[0] in module] logger.info('Only using modules {}'.format(', '.join(module))) elif modules_from_tags: run_modules = [ m for m in run_modules if list(m.keys())[0] in modules_from_tags ] logger.info("Only using modules with '{}' tag".format( ', '.join(module_tag))) if exclude: logger.info("Excluding modules '{}'".format("', '".join(exclude))) if 'general_stats' in exclude: config.skip_generalstats = True exclude = tuple(x for x in exclude if x != 'general_stats') run_modules = [ m for m in run_modules if list(m.keys())[0] not in exclude ] if len(run_modules) == 0: logger.critical('No analysis modules specified!') sys.exit(1) run_module_names = [list(m.keys())[0] for m in run_modules] logger.debug("Analysing modules: {}".format(', '.join(run_module_names))) # Create the temporary working directories tmp_dir = tempfile.mkdtemp() logger.debug( 'Using temporary directory for creating report: {}'.format(tmp_dir)) config.data_tmp_dir = os.path.join(tmp_dir, 'multiqc_data') if filename != 'stdout' and config.make_data_dir == True: config.data_dir = config.data_tmp_dir os.makedirs(config.data_dir) else: config.data_dir = None config.plots_tmp_dir = os.path.join(tmp_dir, 'multiqc_plots') if filename != 'stdout' and config.export_plots == True: config.plots_dir = config.plots_tmp_dir os.makedirs(config.plots_dir) # Load the template template_mod = config.avail_templates[config.template].load() # Add an output subdirectory if specified by template try: config.output_dir = os.path.join(config.output_dir, template_mod.output_subdir) except AttributeError: pass # No subdirectory variable given # Add custom content section names try: if 'custom_content' in run_module_names: run_module_names.extend(config.custom_data.keys()) except AttributeError: pass # custom_data not in config # Get the list of files to search report.get_filelist(run_module_names) # Run the modules! plugin_hooks.mqc_trigger('before_modules') report.modules_output = list() sys_exit_code = 0 for mod_dict in run_modules: try: this_module = list(mod_dict.keys())[0] mod_cust_config = list(mod_dict.values())[0] mod = config.avail_modules[this_module].load() mod.mod_cust_config = mod_cust_config # feels bad doing this, but seems to work output = mod() if type(output) != list: output = [output] for m in output: report.modules_output.append(m) # Copy over css & js files if requested by the theme try: for to, path in report.modules_output[-1].css.items(): copy_to = os.path.join(tmp_dir, to) os.makedirs(os.path.dirname(copy_to)) shutil.copyfile(path, copy_to) except OSError as e: if e.errno == errno.EEXIST: pass else: raise except AttributeError: pass try: for to, path in report.modules_output[-1].js.items(): copy_to = os.path.join(tmp_dir, to) os.makedirs(os.path.dirname(copy_to)) shutil.copyfile(path, copy_to) except OSError as e: if e.errno == errno.EEXIST: pass else: raise except AttributeError: pass except UserWarning: logger.debug("No samples found: {}".format( list(mod_dict.keys())[0])) except KeyboardInterrupt: shutil.rmtree(tmp_dir) logger.critical("User Cancelled Execution!\n{eq}\n{tb}{eq}\n". format(eq=('=' * 60), tb=traceback.format_exc()) + "User Cancelled Execution!\nExiting MultiQC...") sys.exit(1) except: # Flag the error, but carry on logger.error("Oops! The '{}' MultiQC module broke... \n".format(this_module) + \ " Please copy the following traceback and report it at " + \ "https://github.com/ewels/MultiQC/issues \n" + \ " If possible, please include a log file that triggers the error - " + \ "the last file found was:\n" + \ " {}\n".format(report.last_found_file) + \ ('='*60)+"\nModule {} raised an exception: {}".format( this_module, traceback.format_exc()) + ('='*60)) sys_exit_code = 1 # Did we find anything? if len(report.modules_output) == 0: logger.warn("No analysis results found. Cleaning up..") shutil.rmtree(tmp_dir) logger.info("MultiQC complete") # Exit with an error code if a module broke sys.exit(sys_exit_code) # Sort the report sections if we have a config if len(getattr(config, 'report_section_order', {})) > 0: section_id_order = {} idx = 10 for mod in reversed(report.modules_output): section_id_order[mod.anchor] = idx idx += 10 for anchor, ss in config.report_section_order.items(): if anchor not in section_id_order.keys(): continue if ss.get('order') is not None: section_id_order[anchor] = ss['order'] if ss.get('after') in section_id_order.keys(): section_id_order[anchor] = section_id_order[ss['after']] + 1 if ss.get('before') in section_id_order.keys(): section_id_order[anchor] = section_id_order[ss['before']] - 1 sorted_ids = sorted(section_id_order, key=section_id_order.get) report.modules_output = [ mod for i in reversed(sorted_ids) for mod in report.modules_output if mod.anchor == i ] plugin_hooks.mqc_trigger('after_modules') # Remove empty data sections from the General Stats table empty_keys = [ i for i, d in enumerate(report.general_stats_data[:]) if len(d) == 0 ] empty_keys.sort(reverse=True) for i in empty_keys: del report.general_stats_data[i] del report.general_stats_headers[i] # Add general-stats IDs to table row headers for idx, h in enumerate(report.general_stats_headers): for k in h.keys(): if 'rid' not in h[k]: h[k]['rid'] = re.sub(r'\W+', '_', k).strip().strip('_') ns_html = re.sub(r'\W+', '_', h[k]['namespace']).strip().strip('_').lower() report.general_stats_headers[idx][k]['rid'] = report.save_htmlid( 'mqc-generalstats-{}-{}'.format(ns_html, h[k]['rid'])) # Generate the General Statistics HTML & write to file if len(report.general_stats_data) > 0: pconfig = { 'id': 'general_stats_table', 'table_title': 'General Statistics', 'save_file': True, 'raw_data_fn': 'multiqc_general_stats' } report.general_stats_html = table.plot(report.general_stats_data, report.general_stats_headers, pconfig) else: config.skip_generalstats = True # Write the report sources to disk if config.data_dir is not None: report.data_sources_tofile() # Compress the report plot JSON data logger.info("Compressing plot data") report.plot_compressed_json = report.compress_json(report.plot_data) plugin_hooks.mqc_trigger('before_report_generation') # Data Export / MegaQC integration - save report data to file or send report data to an API endpoint if (config.data_dump_file or config.megaqc_url) and config.megaqc_upload: multiqc_json_dump = megaqc.multiqc_dump_json(report) if config.data_dump_file: util_functions.write_data_file(multiqc_json_dump, 'multiqc_data', False, 'json') if config.megaqc_url: megaqc.multiqc_api_post(multiqc_json_dump) # Make the final report path & data directories if filename != 'stdout': config.output_fn = os.path.join(config.output_dir, config.output_fn_name) config.data_dir = os.path.join(config.output_dir, config.data_dir_name) # Check for existing reports and remove if -f was specified if os.path.exists( config.output_fn) or (config.make_data_dir and os.path.exists(config.data_dir)): if config.force: if os.path.exists(config.output_fn): logger.warning( "Deleting : {} (-f was specified)".format( os.path.relpath(config.output_fn))) os.remove(config.output_fn) if config.make_data_dir and os.path.exists(config.data_dir): logger.warning( "Deleting : {} (-f was specified)".format( os.path.relpath(config.data_dir))) shutil.rmtree(config.data_dir) else: # Set up the base names of the report and the data dir report_num = 1 report_base, report_ext = os.path.splitext( config.output_fn_name) dir_base = os.path.basename(config.data_dir) # Iterate through appended numbers until we find one that's free while os.path.exists( config.output_fn) or (config.make_data_dir and os.path.exists(config.data_dir)): config.output_fn = os.path.join( config.output_dir, "{}_{}{}".format(report_base, report_num, report_ext)) config.data_dir = os.path.join( config.output_dir, "{}_{}".format(dir_base, report_num)) report_num += 1 config.output_fn_name = os.path.basename(config.output_fn) config.data_dir_name = os.path.basename(config.data_dir) logger.warning( "Previous MultiQC output found! Adjusting filenames..") logger.warning( "Use -f or --force to overwrite existing reports instead") # Make directories for report if needed if not os.path.exists(os.path.dirname(config.output_fn)): os.makedirs(os.path.dirname(config.output_fn)) logger.info("Report : {}".format(os.path.relpath( config.output_fn))) if config.make_data_dir == False: logger.info("Data : None") else: # Make directories for data_dir logger.info("Data : {}".format( os.path.relpath(config.data_dir))) if not os.path.exists(config.data_dir): os.makedirs(config.data_dir) # Modules have run, so data directory should be complete by now. Move its contents. for f in os.listdir(config.data_tmp_dir): fn = os.path.join(config.data_tmp_dir, f) logger.debug("Moving data file from '{}' to '{}'".format( fn, config.data_dir)) shutil.move(fn, config.data_dir) # Copy across the static plot images if requested if config.export_plots: config.plots_dir = os.path.join(config.output_dir, config.plots_dir_name) if os.path.exists(config.plots_dir): if config.force: logger.warning( "Deleting : {} (-f was specified)".format( os.path.relpath(config.plots_dir))) shutil.rmtree(config.plots_dir) else: logger.error("Output directory {} already exists.".format( config.plots_dir)) logger.info( "Use -f or --force to overwrite existing reports") shutil.rmtree(tmp_dir) sys.exit(1) os.makedirs(config.plots_dir) logger.info("Plots : {}".format( os.path.relpath(config.plots_dir))) # Modules have run, so plots directory should be complete by now. Move its contents. for f in os.listdir(config.plots_tmp_dir): fn = os.path.join(config.plots_tmp_dir, f) logger.debug("Moving plots directory from '{}' to '{}'".format( fn, config.plots_dir)) shutil.move(fn, config.plots_dir) plugin_hooks.mqc_trigger('before_template') # Load in parent template files first if a child theme try: parent_template = config.avail_templates[ template_mod.template_parent].load() copy_tree(parent_template.template_dir, tmp_dir) except AttributeError: pass # Not a child theme # Copy the template files to the tmp directory (distutils overwrites parent theme files) copy_tree(template_mod.template_dir, tmp_dir) # Function to include file contents in Jinja template def include_file(name, fdir=tmp_dir, b64=False): try: if fdir is None: fdir = '' if b64: with io.open(os.path.join(fdir, name), "rb") as f: return base64.b64encode(f.read()).decode('utf-8') else: with io.open(os.path.join(fdir, name), "r", encoding='utf-8') as f: return f.read() except (OSError, IOError) as e: logger.error("Could not include file '{}': {}".format(name, e)) # Load the report template try: env = jinja2.Environment(loader=jinja2.FileSystemLoader(tmp_dir)) env.globals['include_file'] = include_file j_template = env.get_template(template_mod.base_fn) except: raise IOError("Could not load {} template file '{}'".format( config.template, template_mod.base_fn)) # Use jinja2 to render the template and overwrite config.analysis_dir = [os.path.realpath(d) for d in config.analysis_dir] report_output = j_template.render(report=report, config=config) if filename == 'stdout': print(report_output.encode('utf-8'), file=sys.stdout) else: try: with io.open(config.output_fn, "w", encoding='utf-8') as f: print(report_output, file=f) except IOError as e: raise IOError("Could not print report to '{}' - {}".format( config.output_fn, IOError(e))) # Copy over files if requested by the theme try: for f in template_mod.copy_files: fn = os.path.join(tmp_dir, f) dest_dir = os.path.join(os.path.dirname(config.output_fn), f) copy_tree(fn, dest_dir) except AttributeError: pass # No files to copy # Clean up temporary directory shutil.rmtree(tmp_dir) # Zip the data directory if requested if config.zip_data_dir and config.data_dir is not None: shutil.make_archive(config.data_dir, 'zip', config.data_dir) shutil.rmtree(config.data_dir) # Try to create a PDF if requested if make_pdf: try: pdf_fn_name = config.output_fn.replace('.html', '.pdf') pandoc_call = [ 'pandoc', '--standalone', config.output_fn, '--output', pdf_fn_name, '--pdf-engine=xelatex', '-V', 'documentclass=article', '-V', 'geometry=margin=1in', '-V', 'title=' ] if config.pandoc_template is not None: pandoc_call.append('--template={}'.format( config.pandoc_template)) logger.debug( "Attempting Pandoc conversion to PDF with following command:\n{}" .format(' '.join(pandoc_call))) pdf_exit_code = subprocess.call(pandoc_call) if pdf_exit_code != 0: logger.error( "Error creating PDF! Pandoc returned a non-zero exit code." ) else: logger.info("PDF Report : {}".format(pdf_fn_name)) except OSError as e: if e.errno == os.errno.ENOENT: logger.error( 'Error creating PDF - pandoc not found. Is it installed? http://pandoc.org/' ) else: logger.error( "Error creating PDF! Something went wrong when creating the PDF\n" + ('=' * 60) + "\n{}\n".format(traceback.format_exc()) + ('=' * 60)) plugin_hooks.mqc_trigger('execution_finish') logger.info("MultiQC complete") if lint and len(report.lint_errors) > 0: logger.error("Found {} linting errors!\n{}".format( len(report.lint_errors), "\n".join(report.lint_errors))) sys_exit_code = 1 # Move the log file into the data directory log.move_tmp_log(logger) # Exit with an error code if a module broke sys.exit(sys_exit_code)
def matplotlib_bargraph(plotdata, plotsamples, pconfig=None): """ Plot a bargraph with Matplot lib and return a HTML string. Either embeds a base64 encoded image within HTML or writes the plot and links to it. Should be called by plot_bargraph, which properly formats the input data. """ if pconfig is None: pconfig = {} # Plot group ID if pconfig.get('id') is None: pconfig['id'] = 'mqc_mplplot_' + ''.join(random.sample(letters, 10)) # Sanitise plot ID and check for duplicates pconfig['id'] = report.save_htmlid(pconfig['id']) # Individual plot IDs pids = [] for k in range(len(plotdata)): try: name = pconfig['data_labels'][k] except: name = k + 1 pid = 'mqc_{}_{}'.format(pconfig['id'], name) pid = report.save_htmlid(pid, skiplint=True) pids.append(pid) html = '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> ' + \ 'Flat image plot. Toolbox functions such as highlighting / hiding samples will not work ' + \ '(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>' html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig['id']) # Same defaults as HighCharts for consistency default_colors = [ '#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1' ] # Counts / Percentages Switch if pconfig.get('cpswitch') is not False and not config.simple_output: if pconfig.get('cpswitch_c_active', True) is True: c_active = 'active' p_active = '' else: c_active = '' p_active = 'active' pconfig['stacking'] = 'percent' c_label = pconfig.get('cpswitch_counts_label', 'Counts') p_label = pconfig.get('cpswitch_percent_label', 'Percentages') html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_setcountspcnt"> \n\ <button class="btn btn-default btn-sm {c_a} counts">{c_l}</button> \n\ <button class="btn btn-default btn-sm {p_a} pcnt">{p_l}</button> \n\ </div> '.format(c_a=c_active, p_a=p_active, c_l=c_label, p_l=p_label) if len(plotdata) > 1: html += ' ' # Buttons to cycle through different datasets if len(plotdata) > 1 and not config.simple_output: html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n' for k, p in enumerate(plotdata): pid = pids[k] active = 'active' if k == 0 else '' try: name = pconfig['data_labels'][k] except: name = k + 1 html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format( a=active, pid=pid, n=name) html += '</div>\n\n' # Go through datasets creating plots for pidx, pdata in enumerate(plotdata): # Save plot data to file fdata = {} for d in pdata: for didx, dval in enumerate(d['data']): s_name = plotsamples[pidx][didx] if s_name not in fdata: fdata[s_name] = dict() fdata[s_name][d['name']] = dval util_functions.write_data_file(fdata, pids[pidx]) # Plot percentage as well as counts plot_pcts = [False] if pconfig.get('cpswitch') is not False: plot_pcts = [False, True] # Switch out NaN for 0s so that MatPlotLib doesn't ignore stuff for idx, d in enumerate(pdata): pdata[idx]['data'] = [ x if not math.isnan(x) else 0 for x in d['data'] ] for plot_pct in plot_pcts: # Plot ID pid = pids[pidx] hide_plot = False if plot_pct is True: pid = '{}_pc'.format(pid) if pconfig.get('cpswitch_c_active', True) is True: hide_plot = True else: if pconfig.get('cpswitch_c_active', True) is not True: hide_plot = True # Set up figure plt_height = len(plotsamples[pidx]) / 2.3 plt_height = max(6, plt_height) # At least 6" tall plt_height = min(30, plt_height) # Cap at 30" tall bar_width = 0.8 fig = plt.figure(figsize=(14, plt_height), frameon=False) axes = fig.add_subplot(111) y_ind = range(len(plotsamples[pidx])) # Count totals for each sample if plot_pct is True: s_totals = [0 for _ in pdata[0]['data']] for series_idx, d in enumerate(pdata): for sample_idx, v in enumerate(d['data']): s_totals[sample_idx] += v # Plot bars dlabels = [] for idx, d in enumerate(pdata): # Plot percentages values = d['data'] if len(values) < len(y_ind): values.extend([0] * (len(y_ind) - len(values))) if plot_pct is True: for (key, var) in enumerate(values): s_total = s_totals[key] if s_total == 0: values[key] = 0 else: values[key] = (float(var + 0.0) / float(s_total)) * 100 # Get offset for stacked bars if idx == 0: prevdata = [0] * len(plotsamples[pidx]) else: for i, p in enumerate(prevdata): prevdata[i] += pdata[idx - 1]['data'][i] # Default colour index cidx = idx while cidx >= len(default_colors): cidx -= len(default_colors) # Save the name of this series dlabels.append(d['name']) # Add the series of bars to the plot axes.barh(y_ind, values, bar_width, left=prevdata, color=d.get('color', default_colors[cidx]), align='center', linewidth=pconfig.get('borderWidth', 0)) # Tidy up axes axes.tick_params(labelsize=8, direction='out', left=False, right=False, top=False, bottom=False) axes.set_xlabel( pconfig.get('ylab', '') ) # I know, I should fix the fact that the config is switched axes.set_ylabel(pconfig.get('xlab', '')) axes.set_yticks(y_ind) # Specify where to put the labels axes.set_yticklabels( plotsamples[pidx]) # Set y axis sample name labels axes.set_ylim( (-0.5, len(y_ind) - 0.5)) # Reduce padding around plot area if plot_pct is True: axes.set_xlim((0, 100)) # Add percent symbols vals = axes.get_xticks() axes.set_xticklabels(['{:.0f}%'.format(x) for x in vals]) else: default_xlimits = axes.get_xlim() axes.set_xlim((pconfig.get('ymin', default_xlimits[0]), pconfig.get('ymax', default_xlimits[1]))) if 'title' in pconfig: top_gap = 1 + (0.5 / plt_height) plt.text(0.5, top_gap, pconfig['title'], horizontalalignment='center', fontsize=16, transform=axes.transAxes) axes.grid(True, zorder=0, which='both', axis='x', linestyle='-', color='#dedede', linewidth=1) axes.set_axisbelow(True) axes.spines['right'].set_visible(False) axes.spines['top'].set_visible(False) axes.spines['bottom'].set_visible(False) axes.spines['left'].set_visible(False) plt.gca().invert_yaxis() # y axis is reverse sorted otherwise # Hide some labels if we have a lot of samples show_nth = max(1, math.ceil(len(pdata[0]['data']) / 150)) for idx, label in enumerate(axes.get_yticklabels()): if idx % show_nth != 0: label.set_visible(False) # Legend bottom_gap = -1 * (1 - ((plt_height - 1.5) / plt_height)) lgd = axes.legend(dlabels, loc='lower center', bbox_to_anchor=(0, bottom_gap, 1, .102), ncol=5, mode='expand', fontsize=8, frameon=False) # Should this plot be hidden on report load? hidediv = '' if pidx > 0 or hide_plot: hidediv = ' style="display:none;"' # Save the plot to the data directory if export is requested if config.export_plots: for fformat in config.export_plot_formats: # Make the directory if it doesn't already exist plot_dir = os.path.join(config.plots_dir, fformat) if not os.path.exists(plot_dir): os.makedirs(plot_dir) # Save the plot plot_fn = os.path.join(plot_dir, '{}.{}'.format(pid, fformat)) fig.savefig(plot_fn, format=fformat, bbox_extra_artists=(lgd, ), bbox_inches='tight') # Output the figure to a base64 encoded string if getattr(get_template_mod(), 'base64_plots', True) is True: img_buffer = io.BytesIO() fig.savefig(img_buffer, format='png', bbox_inches='tight') b64_img = base64.b64encode( img_buffer.getvalue()).decode('utf8') img_buffer.close() html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format( pid, hidediv, b64_img) # Link to the saved image else: plot_relpath = os.path.join(config.plots_dir_name, 'png', '{}.png'.format(pid)) html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format( pid, hidediv, plot_relpath) plt.close(fig) # Close wrapping div html += '</div>' report.num_mpl_plots += 1 return html
def general_stats_sample_meta(self): """ Add metadata about each sample to the General Stats table """ meta = report.ngi.get('sample_meta') if meta is not None and len(meta) > 0: log.info('Found {} samples in StatusDB'.format(len(meta))) # Write to file util_functions.write_data_file(meta, 'ngi_meta') # Add to General Stats table gsdata = dict() formats = dict() s_names = dict() ngi_ids = dict() conc_units = '' for sid in meta: # Find first sample name matching this sample ID s_name = None for x in sorted(self.s_names): if sid in x: s_name = x s_names[s_name] = x ngi_ids[s_name] = sid break # Skip this sample if we don't have any matching data if s_name is None: log.debug("Skipping StatusDB metadata for sample {} as no bioinfo report logs found.".format(sid)) continue # Make a dict to hold new data for General Stats gsdata[s_name] = dict() # NGI name try: gsdata[s_name]['user_sample_name'] = report.ngi['ngi_names'][ngi_ids[s_name]] except KeyError: pass # RIN score try: gsdata[s_name]['initial_qc_rin'] = meta[sid]['initial_qc']['rin'] except KeyError: pass # Try to figure out which library prep was used seq_lp = None for lp in sorted(meta[sid].get('library_prep', {}).keys()): try: if len(meta[sid]['library_prep'][lp]['sample_run_metrics']) > 0: if seq_lp is None: seq_lp = lp else: seq_lp = None log.warn('Found multiple sequenced lib preps for {} - skipping metadata'.format(sid)) break except KeyError: pass if seq_lp is not None: try: if meta[sid]['library_prep'][lp]['amount_taken_(ng)'] is not None: gsdata[s_name]['amount_taken'] = meta[sid]['library_prep'][lp]['amount_taken_(ng)'] except KeyError: pass try: for lv in sorted(meta[sid]['library_prep'][lp]['library_validation'].keys()): gsdata[s_name]['lp_concentration'] = meta[sid]['library_prep'][lp]['library_validation'][lv]['concentration'] formats[s_name] = meta[sid]['library_prep'][lp]['library_validation'][lv]['conc_units'] except KeyError: pass log.info("Matched meta for {} samples from StatusDB with report sample names".format(len(s_names))) if len(s_names) == 0: return None # Deal with having more than one initial QC concentration unit formats_set = set(formats.values()) if len(formats_set) > 1: log.warning("Mixture of library_validation concentration units! Found: {}".format(", ".join(formats_set))) for s_name in gsdata: try: gsdata[s_name]['lp_concentration'] = '{} {}'.format(gsdata[s_name]['lp_concentration'], formats[s_name]) except KeyError: pass elif len(formats_set) == 1: conc_units = formats_set.pop() # Decide on whether to show or hide conc & amount taken based on range conc_hidden = True amounts_hidden = True try: concs = [gsdata[x]['lp_concentration'] for x in gsdata] if max(concs) - min(concs) > 50: conc_hidden = False except (KeyError, ValueError): conc_hidden = False try: amounts = [gsdata[x]['amount_taken'] for x in gsdata] if max(amounts) - min(amounts) > 10: amounts_hidden = False except KeyError: amounts_hidden = False # Prepend columns to the General Stats table (far left) gsheaders_prepend = OrderedDict() gsheaders_prepend['user_sample_name'] = { 'namespace': 'NGI', 'title': 'Name', 'description': 'User sample ID', 'scale': False } report.general_stats_data.insert(0, gsdata) report.general_stats_headers.insert(0, gsheaders_prepend) # Add columns to the far right of the General Stats table gsheaders = OrderedDict() gsheaders['initial_qc_rin'] = { 'namespace': 'NGI', 'title': 'RIN', 'description': 'Initial QC: RNA Integrity Number', 'min': 0, 'max': 10, 'scale': 'YlGn', 'format': '{:,.2f}' } gsheaders['lp_concentration'] = { 'namespace': 'NGI', 'title': 'Lib Conc. ({})'.format(conc_units), 'description': 'Library Prep: Concentration ({})'.format(conc_units), 'min': 0, 'scale': 'YlGn', 'format': '{:.,0f}', 'hidden': conc_hidden } gsheaders['amount_taken'] = { 'namespace': 'NGI', 'title': 'Amount Taken (ng)', 'description': 'Library Prep: Amount Taken (ng)', 'min': 0, 'scale': 'YlGn', 'format': '{:.,0f}', 'hidden': amounts_hidden } report.general_stats_data.append(gsdata) report.general_stats_headers.append(gsheaders)
def matplotlib_linegraph (plotdata, pconfig={}): """ Plot a line graph with Matplot lib and return a HTML string. Either embeds a base64 encoded image within HTML or writes the plot and links to it. Should be called by plot_bargraph, which properly formats the input data. """ # Plot group ID if pconfig.get('id') is None: pconfig['id'] = 'mqc_mplplot_'+''.join(random.sample(letters, 10)) # Individual plot IDs pids = [] for k in range(len(plotdata)): try: name = pconfig['data_labels'][k]['name'] except: name = k+1 pid = 'mqc_{}_{}'.format(pconfig['id'], name) pid = "".join([c for c in pid if c.isalpha() or c.isdigit() or c == '_' or c == '-']) pids.append(pid) html = '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> ' + \ 'Flat image plot. Toolbox functions such as highlighting / hiding samples will not work ' + \ '(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>' html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig['id']) # Same defaults as HighCharts for consistency default_colors = ['#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1'] # Buttons to cycle through different datasets if len(plotdata) > 1 and not config.simple_output: html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n' for k, p in enumerate(plotdata): pid = pids[k] active = 'active' if k == 0 else '' try: name = pconfig['data_labels'][k]['name'] except: name = k+1 html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format(a=active, pid=pid, n=name) html += '</div>\n\n' # Go through datasets creating plots for pidx, pdata in enumerate(plotdata): # Plot ID pid = pids[pidx] # Save plot data to file fdata = OrderedDict() lastcats = None sharedcats = True for d in pdata: fdata[d['name']] = OrderedDict() for i, x in enumerate(d['data']): if type(x) is list: fdata[d['name']][str(x[0])] = x[1] # Check to see if all categories are the same if lastcats is None: lastcats = [x[0] for x in d['data']] elif lastcats != [x[0] for x in d['data']]: sharedcats = False else: try: fdata[d['name']][pconfig['categories'][i]] = x except (KeyError, IndexError): fdata[d['name']][str(i)] = x # Custom tsv output if the x axis varies if not sharedcats and config.data_format == 'tsv': fout = '' for d in pdata: fout += "\t"+"\t".join([str(x[0]) for x in d['data']]) fout += "\n{}\t".format(d['name']) fout += "\t".join([str(x[1]) for x in d['data']]) fout += "\n" with io.open (os.path.join(config.data_dir, '{}.txt'.format(pid)), 'w', encoding='utf-8') as f: print( fout.encode('utf-8', 'ignore').decode('utf-8'), file=f ) else: util_functions.write_data_file(fdata, pid) # Set up figure fig = plt.figure(figsize=(14, 6), frameon=False) axes = fig.add_subplot(111) # Go through data series for idx, d in enumerate(pdata): # Default colour index cidx = idx while cidx >= len(default_colors): cidx -= len(default_colors) # Line style linestyle = 'solid' if d.get('dashStyle', None) == 'Dash': linestyle = 'dashed' # Reformat data (again) try: axes.plot([x[0] for x in d['data']], [x[1] for x in d['data']], label=d['name'], color=d.get('color', default_colors[cidx]), linestyle=linestyle, linewidth=1, marker=None) except TypeError: # Categorical data on x axis axes.plot(d['data'], label=d['name'], color=d.get('color', default_colors[cidx]), linewidth=1, marker=None) # Tidy up axes axes.tick_params(labelsize=8, direction='out', left=False, right=False, top=False, bottom=False) axes.set_xlabel(pconfig.get('xlab', '')) axes.set_ylabel(pconfig.get('ylab', '')) # Dataset specific y label try: axes.set_ylabel(pconfig['data_labels'][pidx]['ylab']) except: pass # Axis limits default_ylimits = axes.get_ylim() ymin = default_ylimits[0] if 'ymin' in pconfig: ymin = pconfig['ymin'] elif 'yCeiling' in pconfig: ymin = min(pconfig['yCeiling'], default_ylimits[0]) ymax = default_ylimits[1] if 'ymax' in pconfig: ymax = pconfig['ymax'] elif 'yFloor' in pconfig: ymax = max(pconfig['yCeiling'], default_ylimits[1]) if (ymax - ymin) < pconfig.get('yMinRange', 0): ymax = ymin + pconfig['yMinRange'] axes.set_ylim((ymin, ymax)) # Dataset specific ymax try: axes.set_ylim((ymin, pconfig['data_labels'][pidx]['ymax'])) except: pass default_xlimits = axes.get_xlim() xmin = default_xlimits[0] if 'xmin' in pconfig: xmin = pconfig['xmin'] elif 'xCeiling' in pconfig: xmin = min(pconfig['xCeiling'], default_xlimits[0]) xmax = default_xlimits[1] if 'xmax' in pconfig: xmax = pconfig['xmax'] elif 'xFloor' in pconfig: xmax = max(pconfig['xCeiling'], default_xlimits[1]) if (xmax - xmin) < pconfig.get('xMinRange', 0): xmax = xmin + pconfig['xMinRange'] axes.set_xlim((xmin, xmax)) # Plot title if 'title' in pconfig: plt.text(0.5, 1.05, pconfig['title'], horizontalalignment='center', fontsize=16, transform=axes.transAxes) axes.grid(True, zorder=10, which='both', axis='y', linestyle='-', color='#dedede', linewidth=1) # X axis categories, if specified if 'categories' in pconfig: axes.set_xticks([i for i,v in enumerate(pconfig['categories'])]) axes.set_xticklabels(pconfig['categories']) # Axis lines xlim = axes.get_xlim() axes.plot([xlim[0], xlim[1]], [0, 0], linestyle='-', color='#dedede', linewidth=2) axes.set_axisbelow(True) axes.spines['right'].set_visible(False) axes.spines['top'].set_visible(False) axes.spines['bottom'].set_visible(False) axes.spines['left'].set_visible(False) # Background colours, if specified if 'yPlotBands' in pconfig: xlim = axes.get_xlim() for pb in pconfig['yPlotBands']: axes.barh(pb['from'], xlim[1], height = pb['to']-pb['from'], left=xlim[0], color=pb['color'], linewidth=0, zorder=0) if 'xPlotBands' in pconfig: ylim = axes.get_ylim() for pb in pconfig['xPlotBands']: axes.bar(pb['from'], ylim[1], width = pb['to']-pb['from'], bottom=ylim[0], color=pb['color'], linewidth=0, zorder=0) # Tight layout - makes sure that legend fits in and stuff if len(pdata) <= 15: lgd = axes.legend(loc='lower center', bbox_to_anchor=(0, -0.22, 1, .102), ncol=5, mode='expand', fontsize=8, frameon=False) plt.tight_layout(rect=[0,0.08,1,0.92]) else: plt.tight_layout(rect=[0,0,1,0.92]) # Should this plot be hidden on report load? hidediv = '' if pidx > 0: hidediv = ' style="display:none;"' # Save the plot to the data directory if export is requests if config.export_plots: for fformat in config.export_plot_formats: # Make the directory if it doesn't already exist plot_dir = os.path.join(config.plots_dir, fformat) if not os.path.exists(plot_dir): os.makedirs(plot_dir) # Save the plot plot_fn = os.path.join(plot_dir, '{}.{}'.format(pid, fformat)) fig.savefig(plot_fn, format=fformat, bbox_inches='tight') # Output the figure to a base64 encoded string if getattr(get_template_mod(), 'base64_plots', True) is True: img_buffer = io.BytesIO() fig.savefig(img_buffer, format='png', bbox_inches='tight') b64_img = base64.b64encode(img_buffer.getvalue()).decode('utf8') img_buffer.close() html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format(pid, hidediv, b64_img) # Save to a file and link <img> else: plot_relpath = os.path.join(config.plots_dir_name, 'png', '{}.png'.format(pid)) html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(pid, hidediv, plot_relpath) plt.close(fig) # Close wrapping div html += '</div>' report.num_mpl_plots += 1 return html
def write_data_file(self, data, fn, sort_cols=False, data_format=None): """ Redirects to report.write_data_file() """ util_functions.write_data_file(data, fn, sort_cols, data_format)
def general_stats_sample_meta(self): """ Add metadata about each sample to the General Stats table """ meta = report.ngi.get('sample_meta') if meta is not None and len(meta) > 0: log.info('Found {} samples in StatusDB'.format(len(meta))) # Write to file util_functions.write_data_file(meta, 'ngi_meta') # Add to General Stats table gsdata = dict() formats = dict() s_names = dict() ngi_ids = dict() conc_units = '' for sid in meta: # Find first sample name matching this sample ID s_name = None for x in sorted(self.s_names): if sid in x: s_name = x s_names[s_name] = x ngi_ids[s_name] = sid break # Skip this sample if we don't have any matching data if s_name is None: log.debug("Skipping StatusDB metadata for sample {} as no bioinfo report logs found.".format(sid)) continue # Make a dict to hold new data for General Stats gsdata[s_name] = dict() # NGI name try: gsdata[s_name]['user_sample_name'] = report.ngi['ngi_names'][ngi_ids[s_name]] except KeyError: pass # RIN score try: gsdata[s_name]['initial_qc_rin'] = meta[sid]['initial_qc']['rin'] except KeyError: pass # Try to figure out which library prep was used seq_lp = None for lp in sorted(meta[sid].get('library_prep', {}).keys()): try: if len(meta[sid]['library_prep'][lp]['sample_run_metrics']) > 0: if seq_lp is None: seq_lp = lp else: seq_lp = None log.warn('Found multiple sequenced lib preps for {} - skipping metadata'.format(sid)) break except KeyError: pass if seq_lp is not None: try: if meta[sid]['library_prep'][lp]['amount_taken_(ng)'] is not None: gsdata[s_name]['amount_taken'] = meta[sid]['library_prep'][lp]['amount_taken_(ng)'] except KeyError: pass try: for lv in sorted(meta[sid]['library_prep'][lp]['library_validation'].keys()): gsdata[s_name]['lp_concentration'] = meta[sid]['library_prep'][lp]['library_validation'][lv]['concentration'] formats[s_name] = meta[sid]['library_prep'][lp]['library_validation'][lv]['conc_units'] except KeyError: pass log.info("Matched meta for {} samples from StatusDB with report sample names".format(len(s_names))) if len(s_names) == 0: return None # Deal with having more than one initial QC concentration unit formats_set = set(formats.values()) if len(formats_set) > 1: log.warning("Mixture of library_validation concentration units! Found: {}".format(", ".join(formats_set))) for s_name in gsdata: try: gsdata[s_name]['lp_concentration'] = '{} {}'.format(gsdata[s_name]['lp_concentration'], formats[s_name]) except KeyError: pass elif len(formats_set) == 1: conc_units = formats_set.pop() # Decide on whether to show or hide conc & amount taken based on range conc_hidden = True amounts_hidden = True try: concs = [gsdata[x]['lp_concentration'] for x in gsdata] if max(concs) - min(concs) > 50: conc_hidden = False except (KeyError, ValueError): conc_hidden = False try: amounts = [gsdata[x]['amount_taken'] for x in gsdata] if max(amounts) - min(amounts) > 10: amounts_hidden = False except KeyError: amounts_hidden = False # Prepend columns to the General Stats table (far left) gsheaders_prepend = OrderedDict() gsheaders_prepend['user_sample_name'] = { 'namespace': 'NGI', 'title': 'Name', 'description': 'User sample ID', 'scale': False } report.general_stats_data.insert(0, gsdata) report.general_stats_headers.insert(0, gsheaders_prepend) # Add columns to the far right of the General Stats table gsheaders = OrderedDict() gsheaders['initial_qc_rin'] = { 'namespace': 'NGI', 'title': 'RIN', 'description': 'Initial QC: RNA Integrity Number', 'min': 0, 'max': 10, 'scale': 'YlGn', 'format': '{:.2f}' } gsheaders['lp_concentration'] = { 'namespace': 'NGI', 'title': 'Lib Conc. ({})'.format(conc_units), 'description': 'Library Prep: Concentration ({})'.format(conc_units), 'min': 0, 'scale': 'YlGn', 'format': '{:.0f}', 'hidden': conc_hidden } gsheaders['amount_taken'] = { 'namespace': 'NGI', 'title': 'Amount Taken (ng)', 'description': 'Library Prep: Amount Taken (ng)', 'min': 0, 'scale': 'YlGn', 'format': '{:.0f}', 'hidden': amounts_hidden } report.general_stats_data.append(gsdata) report.general_stats_headers.append(gsheaders)
def matplotlib_linegraph (plotdata, pconfig=None): """ Plot a line graph with Matplot lib and return a HTML string. Either embeds a base64 encoded image within HTML or writes the plot and links to it. Should be called by plot_bargraph, which properly formats the input data. """ if pconfig is None: pconfig = {} # Plot group ID if pconfig.get('id') is None: pconfig['id'] = 'mqc_mplplot_'+''.join(random.sample(letters, 10)) # Sanitise plot ID and check for duplicates pconfig['id'] = report.save_htmlid(pconfig['id']) # Individual plot IDs pids = [] for k in range(len(plotdata)): try: name = pconfig['data_labels'][k]['name'] except: name = k+1 pid = 'mqc_{}_{}'.format(pconfig['id'], name) pid = report.save_htmlid(pid) pids.append(pid) html = '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> ' + \ 'Flat image plot. Toolbox functions such as highlighting / hiding samples will not work ' + \ '(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>' html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig['id']) # Same defaults as HighCharts for consistency default_colors = ['#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1'] # Buttons to cycle through different datasets if len(plotdata) > 1 and not config.simple_output: html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n' for k, p in enumerate(plotdata): pid = pids[k] active = 'active' if k == 0 else '' try: name = pconfig['data_labels'][k]['name'] except: name = k+1 html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format(a=active, pid=pid, n=name) html += '</div>\n\n' # Go through datasets creating plots for pidx, pdata in enumerate(plotdata): # Plot ID pid = pids[pidx] # Save plot data to file fdata = OrderedDict() lastcats = None sharedcats = True for d in pdata: fdata[d['name']] = OrderedDict() for i, x in enumerate(d['data']): if type(x) is list: fdata[d['name']][str(x[0])] = x[1] # Check to see if all categories are the same if lastcats is None: lastcats = [x[0] for x in d['data']] elif lastcats != [x[0] for x in d['data']]: sharedcats = False else: try: fdata[d['name']][pconfig['categories'][i]] = x except (KeyError, IndexError): fdata[d['name']][str(i)] = x # Custom tsv output if the x axis varies if not sharedcats and config.data_format == 'tsv': fout = '' for d in pdata: fout += "\t"+"\t".join([str(x[0]) for x in d['data']]) fout += "\n{}\t".format(d['name']) fout += "\t".join([str(x[1]) for x in d['data']]) fout += "\n" with io.open (os.path.join(config.data_dir, '{}.txt'.format(pid)), 'w', encoding='utf-8') as f: print( fout.encode('utf-8', 'ignore').decode('utf-8'), file=f ) else: util_functions.write_data_file(fdata, pid) # Set up figure fig = plt.figure(figsize=(14, 6), frameon=False) axes = fig.add_subplot(111) # Go through data series for idx, d in enumerate(pdata): # Default colour index cidx = idx while cidx >= len(default_colors): cidx -= len(default_colors) # Line style linestyle = 'solid' if d.get('dashStyle', None) == 'Dash': linestyle = 'dashed' # Reformat data (again) try: axes.plot([x[0] for x in d['data']], [x[1] for x in d['data']], label=d['name'], color=d.get('color', default_colors[cidx]), linestyle=linestyle, linewidth=1, marker=None) except TypeError: # Categorical data on x axis axes.plot(d['data'], label=d['name'], color=d.get('color', default_colors[cidx]), linewidth=1, marker=None) # Tidy up axes axes.tick_params(labelsize=8, direction='out', left=False, right=False, top=False, bottom=False) axes.set_xlabel(pconfig.get('xlab', '')) axes.set_ylabel(pconfig.get('ylab', '')) # Dataset specific y label try: axes.set_ylabel(pconfig['data_labels'][pidx]['ylab']) except: pass # Axis limits default_ylimits = axes.get_ylim() ymin = default_ylimits[0] if 'ymin' in pconfig: ymin = pconfig['ymin'] elif 'yCeiling' in pconfig: ymin = min(pconfig['yCeiling'], default_ylimits[0]) ymax = default_ylimits[1] if 'ymax' in pconfig: ymax = pconfig['ymax'] elif 'yFloor' in pconfig: ymax = max(pconfig['yCeiling'], default_ylimits[1]) if (ymax - ymin) < pconfig.get('yMinRange', 0): ymax = ymin + pconfig['yMinRange'] axes.set_ylim((ymin, ymax)) # Dataset specific ymax try: axes.set_ylim((ymin, pconfig['data_labels'][pidx]['ymax'])) except: pass default_xlimits = axes.get_xlim() xmin = default_xlimits[0] if 'xmin' in pconfig: xmin = pconfig['xmin'] elif 'xCeiling' in pconfig: xmin = min(pconfig['xCeiling'], default_xlimits[0]) xmax = default_xlimits[1] if 'xmax' in pconfig: xmax = pconfig['xmax'] elif 'xFloor' in pconfig: xmax = max(pconfig['xCeiling'], default_xlimits[1]) if (xmax - xmin) < pconfig.get('xMinRange', 0): xmax = xmin + pconfig['xMinRange'] axes.set_xlim((xmin, xmax)) # Plot title if 'title' in pconfig: plt.text(0.5, 1.05, pconfig['title'], horizontalalignment='center', fontsize=16, transform=axes.transAxes) axes.grid(True, zorder=10, which='both', axis='y', linestyle='-', color='#dedede', linewidth=1) # X axis categories, if specified if 'categories' in pconfig: axes.set_xticks([i for i,v in enumerate(pconfig['categories'])]) axes.set_xticklabels(pconfig['categories']) # Axis lines xlim = axes.get_xlim() axes.plot([xlim[0], xlim[1]], [0, 0], linestyle='-', color='#dedede', linewidth=2) axes.set_axisbelow(True) axes.spines['right'].set_visible(False) axes.spines['top'].set_visible(False) axes.spines['bottom'].set_visible(False) axes.spines['left'].set_visible(False) # Background colours, if specified if 'yPlotBands' in pconfig: xlim = axes.get_xlim() for pb in pconfig['yPlotBands']: axes.barh(pb['from'], xlim[1], height = pb['to']-pb['from'], left=xlim[0], color=pb['color'], linewidth=0, zorder=0) if 'xPlotBands' in pconfig: ylim = axes.get_ylim() for pb in pconfig['xPlotBands']: axes.bar(pb['from'], ylim[1], width = pb['to']-pb['from'], bottom=ylim[0], color=pb['color'], linewidth=0, zorder=0) # Tight layout - makes sure that legend fits in and stuff if len(pdata) <= 15: axes.legend(loc='lower center', bbox_to_anchor=(0, -0.22, 1, .102), ncol=5, mode='expand', fontsize=8, frameon=False) plt.tight_layout(rect=[0,0.08,1,0.92]) else: plt.tight_layout(rect=[0,0,1,0.92]) # Should this plot be hidden on report load? hidediv = '' if pidx > 0: hidediv = ' style="display:none;"' # Save the plot to the data directory if export is requests if config.export_plots: for fformat in config.export_plot_formats: # Make the directory if it doesn't already exist plot_dir = os.path.join(config.plots_dir, fformat) if not os.path.exists(plot_dir): os.makedirs(plot_dir) # Save the plot plot_fn = os.path.join(plot_dir, '{}.{}'.format(pid, fformat)) fig.savefig(plot_fn, format=fformat, bbox_inches='tight') # Output the figure to a base64 encoded string if getattr(get_template_mod(), 'base64_plots', True) is True: img_buffer = io.BytesIO() fig.savefig(img_buffer, format='png', bbox_inches='tight') b64_img = base64.b64encode(img_buffer.getvalue()).decode('utf8') img_buffer.close() html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format(pid, hidediv, b64_img) # Save to a file and link <img> else: plot_relpath = os.path.join(config.plots_dir_name, 'png', '{}.png'.format(pid)) html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(pid, hidediv, plot_relpath) plt.close(fig) # Close wrapping div html += '</div>' report.num_mpl_plots += 1 return html
def make_table (dt): """ Build the HTML needed for a MultiQC table. :param data: MultiQC datatable object """ table_id = dt.pconfig.get('id', 'table_{}'.format(''.join(random.sample(letters, 4))) ) table_id = report.save_htmlid(table_id) t_headers = OrderedDict() t_modal_headers = OrderedDict() t_rows = OrderedDict() dt.raw_vals = defaultdict(lambda: dict()) empty_cells = dict() hidden_cols = 1 table_title = dt.pconfig.get('table_title') if table_title is None: table_title = table_id.replace("_", " ").title() for idx, k, header in dt.get_headers_in_order(): rid = header['rid'] # Build the table header cell shared_key = '' if header.get('shared_key', None) is not None: shared_key = ' data-shared-key={}'.format(header['shared_key']) hide = '' muted = '' checked = ' checked="checked"' if header.get('hidden', False) is True: hide = 'hidden' muted = ' text-muted' checked = '' hidden_cols += 1 data_attr = 'data-dmax="{}" data-dmin="{}" data-namespace="{}" {}' \ .format(header['dmax'], header['dmin'], header['namespace'], shared_key) cell_contents = '<span class="mqc_table_tooltip" title="{}: {}">{}</span>' \ .format(header['namespace'], header['description'], header['title']) t_headers[rid] = '<th id="header_{rid}" class="{rid} {h}" {da}>{c}</th>' \ .format(rid=rid, h=hide, da=data_attr, c=cell_contents) empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format(rid=rid, h=hide) # Build the modal table row t_modal_headers[rid] = """ <tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);"> <td class="sorthandle ui-sortable-handle">||</span></td> <td style="text-align:center;"> <input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}"> </td> <td>{name}</td> <td>{title}</td> <td>{desc}</td> <td>{col_id}</td> <td>{sk}</td> </tr>""".format( rid = rid, muted = muted, checked = checked, tid = table_id, col = header['colour'], name = header['namespace'], title = header['title'], desc = header['description'], col_id = '<code>{}</code>'.format(k), sk = header.get('shared_key', '') ) # Make a colour scale if header['scale'] == False: c_scale = None else: c_scale = mqc_colour.mqc_colour_scale(header['scale'], header['dmin'], header['dmax']) # Add the data table cells for (s_name, samp) in dt.data[idx].items(): if k in samp: val = samp[k] kname = '{}_{}'.format(header['namespace'], rid) dt.raw_vals[s_name][kname] = val if 'modify' in header and callable(header['modify']): val = header['modify'](val) try: dmin = header['dmin'] dmax = header['dmax'] percentage = ((float(val) - dmin) / (dmax - dmin)) * 100 percentage = min(percentage, 100) percentage = max(percentage, 0) except (ZeroDivisionError,ValueError): percentage = 0 try: valstring = str(header['format'].format(val)) except ValueError: try: valstring = str(header['format'].format(float(val))) except ValueError: valstring = str(val) except: valstring = str(val) # This is horrible, but Python locale settings are worse if config.thousandsSep_format is None: config.thousandsSep_format = '<span class="mqc_thousandSep"></span>' if config.decimalPoint_format is None: config.decimalPoint_format = '.' valstring = valstring.replace('.', 'DECIMAL').replace(',', 'THOUSAND') valstring = valstring.replace('DECIMAL', config.decimalPoint_format).replace('THOUSAND', config.thousandsSep_format) # Percentage suffixes etc valstring += header.get('suffix', '') # Conditional formatting cmatches = { cfck: False for cfc in config.table_cond_formatting_colours for cfck in cfc } # Find general rules followed by column-specific rules for cfk in ['all_columns', rid]: if cfk in config.table_cond_formatting_rules: # Loop through match types for ftype in cmatches.keys(): # Loop through array of comparison types for cmp in config.table_cond_formatting_rules[cfk].get(ftype, []): try: # Each comparison should be a dict with single key: val if 's_eq' in cmp and str(cmp['s_eq']).lower() == str(val).lower(): cmatches[ftype] = True if 's_contains' in cmp and str(cmp['s_contains']).lower() in str(val).lower(): cmatches[ftype] = True if 's_ne' in cmp and str(cmp['s_ne']).lower() != str(val).lower(): cmatches[ftype] = True if 'eq' in cmp and float(cmp['eq']) == float(val): cmatches[ftype] = True if 'ne' in cmp and float(cmp['ne']) != float(val): cmatches[ftype] = True if 'gt' in cmp and float(cmp['gt']) < float(val): cmatches[ftype] = True if 'lt' in cmp and float(cmp['lt']) > float(val): cmatches[ftype] = True except: logger.warn("Not able to apply table conditional formatting to '{}' ({})".format(val, cmp)) # Apply HTML in order of config keys bgcol = None for cfc in config.table_cond_formatting_colours: for cfck in cfc: # should always be one, but you never know if cmatches[cfck]: bgcol = cfc[cfck] if bgcol is not None: valstring = '<span class="badge" style="background-color:{}">{}</span>'.format(bgcol, valstring) # Build HTML if not header['scale']: if s_name not in t_rows: t_rows[s_name] = dict() t_rows[s_name][rid] = '<td class="{rid} {h}">{v}</td>'.format(rid=rid, h=hide, v=valstring) else: if c_scale is not None: col = ' background-color:{};'.format(c_scale.get_colour(val)) else: col = '' bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(percentage, col) val_html = '<span class="val">{}</span>'.format(valstring) wrapper_html = '<div class="wrapper">{}{}</div>'.format(bar_html, val_html) if s_name not in t_rows: t_rows[s_name] = dict() t_rows[s_name][rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(rid=rid, h=hide, c=wrapper_html) # Remove header if we don't have any filled cells for it if sum([len(rows) for rows in t_rows.values()]) == 0: t_headers.pop(rid, None) t_modal_headers.pop(rid, None) logger.debug('Removing header {} from general stats table, as no data'.format(k)) # # Put everything together # # Buttons above the table html = '' if not config.simple_output: # Copy Table Button html += """ <button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}"> <span class="glyphicon glyphicon-copy"></span> Copy table </button> """.format(tid=table_id) # Configure Columns Button if len(t_headers) > 1: html += """ <button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal"> <span class="glyphicon glyphicon-th"></span> Configure Columns </button> """.format(tid=table_id) # Sort By Highlight button html += """ <button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;"> <span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight </button> """.format(tid=table_id) # Scatter Plot Button if len(t_headers) > 1: html += """ <button type="button" class="mqc_table_makeScatter btn btn-default btn-sm" data-toggle="modal" data-target="#tableScatterModal" data-table="#{tid}"> <span class="glyphicon glyphicon glyphicon-stats"></span> Plot </button> """.format(tid=table_id) # "Showing x of y columns" text html += """ <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small> """.format(tid=table_id, nrows=len(t_rows), ncols_vis = (len(t_headers)+1)-hidden_cols, ncols=len(t_headers)) # Build the table itself collapse_class = 'mqc-table-collapse' if len(t_rows) > 10 and config.collapse_tables else '' html += """ <div id="{tid}_container" class="mqc_table_container"> <div class="table-responsive mqc-table-responsive {cc}"> <table id="{tid}" class="table table-condensed mqc_table" data-title="{title}"> """.format( tid=table_id, title=table_title, cc=collapse_class) # Build the header row col1_header = dt.pconfig.get('col1_header', 'Sample Name') html += '<thead><tr><th class="rowheader">{}</th>{}</tr></thead>'.format(col1_header, ''.join(t_headers.values())) # Build the table body html += '<tbody>' t_row_keys = t_rows.keys() if dt.pconfig.get('sortRows') is not False: t_row_keys = sorted(t_row_keys) for s_name in t_row_keys: html += '<tr>' # Sample name row header html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format(sn=s_name) for k in t_headers: html += t_rows[s_name].get(k, empty_cells[k]) html += '</tr>' html += '</tbody></table></div>' if len(t_rows) > 10 and config.collapse_tables: html += '<div class="mqc-table-expand"><span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span></div>' html += '</div>' # Build the bootstrap modal to customise columns and order if not config.simple_output: html += """ <!-- MultiQC Table Columns Modal --> <div class="modal fade" id="{tid}_configModal" tabindex="-1"> <div class="modal-dialog modal-lg"> <div class="modal-content"> <div class="modal-header"> <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button> <h4 class="modal-title">{title}: Columns</h4> </div> <div class="modal-body"> <p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p> <p> <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button> <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button> </p> <table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table" data-title="{title}"> <thead> <tr> <th class="sorthandle" style="text-align:center;">Sort</th> <th style="text-align:center;">Visible</th> <th>Group</th> <th>Column</th> <th>Description</th> <th>ID</th> <th>Scale</th> </tr> </thead> <tbody> {trows} </tbody> </table> </div> <div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div> </div> </div> </div>""".format( tid=table_id, title=table_title, trows=''.join(t_modal_headers.values()) ) # Save the raw values to a file if requested if dt.pconfig.get('save_file') is True: fn = dt.pconfig.get('raw_data_fn', 'multiqc_{}'.format(table_id) ) util_functions.write_data_file(dt.raw_vals, fn ) report.saved_raw_data[fn] = dt.raw_vals return html
def make_table(dt): """ Build the HTML needed for a MultiQC table. :param data: MultiQC datatable object """ table_id = dt.pconfig.get( 'id', 'table_{}'.format(''.join(random.sample(letters, 4)))) table_id = report.save_htmlid(table_id) t_headers = OrderedDict() t_modal_headers = OrderedDict() t_rows = OrderedDict() dt.raw_vals = defaultdict(lambda: dict()) empty_cells = dict() hidden_cols = 1 table_title = dt.pconfig.get('table_title') if table_title is None: table_title = table_id.replace("_", " ").title() for idx, k, header in dt.get_headers_in_order(): rid = header['rid'] # Build the table header cell shared_key = '' if header.get('shared_key', None) is not None: shared_key = ' data-shared-key={}'.format(header['shared_key']) hide = '' muted = '' checked = ' checked="checked"' if header.get('hidden', False) is True: hide = 'hidden' muted = ' text-muted' checked = '' hidden_cols += 1 data_attr = 'data-dmax="{}" data-dmin="{}" data-namespace="{}" {}' \ .format(header['dmax'], header['dmin'], header['namespace'], shared_key) cell_contents = '<span class="mqc_table_tooltip" title="{}: {}">{}</span>' \ .format(header['namespace'], header['description'], header['title']) t_headers[rid] = '<th id="header_{rid}" class="{rid} {h}" {da}>{c}</th>' \ .format(rid=rid, h=hide, da=data_attr, c=cell_contents) empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format( rid=rid, h=hide) # Build the modal table row t_modal_headers[rid] = """ <tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);"> <td class="sorthandle ui-sortable-handle">||</span></td> <td style="text-align:center;"> <input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}"> </td> <td>{name}</td> <td>{title}</td> <td>{desc}</td> <td>{col_id}</td> <td>{sk}</td> </tr>""".format(rid=rid, muted=muted, checked=checked, tid=table_id, col=header['colour'], name=header['namespace'], title=header['title'], desc=header['description'], col_id='<code>{}</code>'.format(k), sk=header.get('shared_key', '')) # Make a colour scale if header['scale'] == False: c_scale = None else: c_scale = mqc_colour.mqc_colour_scale(header['scale'], header['dmin'], header['dmax']) # Add the data table cells for (s_name, samp) in dt.data[idx].items(): if k in samp: val = samp[k] kname = '{}_{}'.format(header['namespace'], rid) dt.raw_vals[s_name][kname] = val if 'modify' in header and callable(header['modify']): val = header['modify'](val) try: dmin = header['dmin'] dmax = header['dmax'] percentage = ((float(val) - dmin) / (dmax - dmin)) * 100 percentage = min(percentage, 100) percentage = max(percentage, 0) except (ZeroDivisionError, ValueError): percentage = 0 try: valstring = str(header['format'].format(val)) except ValueError: try: valstring = str(header['format'].format(float(val))) except ValueError: valstring = str(val) except: valstring = str(val) # This is horrible, but Python locale settings are worse if config.thousandsSep_format is None: config.thousandsSep_format = '<span class="mqc_thousandSep"></span>' if config.decimalPoint_format is None: config.decimalPoint_format = '.' valstring = valstring.replace('.', 'DECIMAL').replace( ',', 'THOUSAND') valstring = valstring.replace( 'DECIMAL', config.decimalPoint_format).replace( 'THOUSAND', config.thousandsSep_format) # Percentage suffixes etc valstring += header.get('suffix', '') # Build HTML if s_name not in t_rows: t_rows[s_name] = dict() t_rows[s_name][rid] = '<td class="{rid} {h}">{v}</td>'.format( rid=rid, h=hide, v=valstring) # else: # if c_scale is not None: # col = ' background-color:{};'.format(c_scale.get_colour(val)) # else: # col = '' # bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(percentage, col) # val_html = '<span class="val">{}</span>'.format(valstring) # wrapper_html = '<div class="wrapper">{}{}</div>'.format(bar_html, val_html) # # if s_name not in t_rows: # t_rows[s_name] = dict() # t_rows[s_name][rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(rid=rid, h=hide, c=wrapper_html) # Remove header if we don't have any filled cells for it if sum([len(rows) for rows in t_rows.values()]) == 0: t_headers.pop(rid, None) t_modal_headers.pop(rid, None) logger.debug( 'Removing header {} from general stats table, as no data'. format(k)) # # Put everything together # # Buttons above the table html = '' if not config.simple_output: # Copy Table Button html += """ <button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}"> <span class="glyphicon glyphicon-copy"></span> Copy table </button> """.format(tid=table_id) # Configure Columns Button if len(t_headers) > 1: html += """ <button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal"> <span class="glyphicon glyphicon-th"></span> Configure Columns </button> """.format(tid=table_id) # Sort By Highlight button html += """ <button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;"> <span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight </button> """.format(tid=table_id) # Scatter Plot Button if len(t_headers) > 1: html += """ <button type="button" class="mqc_table_makeScatter btn btn-default btn-sm" data-toggle="modal" data-target="#tableScatterModal" data-table="#{tid}"> <span class="glyphicon glyphicon glyphicon-stats"></span> Plot </button> """.format(tid=table_id) # "Showing x of y columns" text html += """ <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small> """.format(tid=table_id, nrows=len(t_rows), ncols_vis=(len(t_headers) + 1) - hidden_cols, ncols=len(t_headers)) # Build the table itself collapse_class = 'mqc-table-collapse' if len( t_rows) > 10 and config.collapse_tables else '' html += """ <div id="{tid}_container" class="mqc_table_container"> <div class="table-responsive mqc-table-responsive {cc}"> <table id="{tid}" class="table table-condensed mqc_table" data-title="{title}"> """.format(tid=table_id, title=table_title, cc=collapse_class) # Build the header row col1_header = dt.pconfig.get('col1_header', ' ') html += '<thead><tr><th class="rowheader">{}</th>{}</tr></thead>'.format( col1_header, ''.join(t_headers.values())) # Build the table body html += '<tbody>' t_row_keys = t_rows.keys() if dt.pconfig.get('sortRows') is not False: t_row_keys = sorted(t_row_keys) for s_name in t_row_keys: html += '<tr>' # Sample name row header html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format( sn=s_name) for k in t_headers: html += t_rows[s_name].get(k, empty_cells[k]) html += '</tr>' html += '</tbody></table></div>' if len(t_rows) > 10 and config.collapse_tables: html += '<div class="mqc-table-expand"><span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span></div>' html += '</div>' # Build the bootstrap modal to customise columns and order if not config.simple_output: html += """ <!-- MultiQC Table Columns Modal --> <div class="modal fade" id="{tid}_configModal" tabindex="-1"> <div class="modal-dialog modal-lg"> <div class="modal-content"> <div class="modal-header"> <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button> <h4 class="modal-title">{title}: Columns</h4> </div> <div class="modal-body"> <p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p> <p> <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button> <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button> </p> <table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table" data-title="{title}"> <thead> <tr> <th class="sorthandle" style="text-align:center;">Sort</th> <th style="text-align:center;">Visible</th> <th>Group</th> <th>Column</th> <th>Description</th> <th>ID</th> <th>Scale</th> </tr> </thead> <tbody> {trows} </tbody> </table> </div> <div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div> </div> </div> </div>""".format(tid=table_id, title=table_title, trows=''.join(t_modal_headers.values())) # Save the raw values to a file if requested if dt.pconfig.get('save_file') is True: fn = dt.pconfig.get('raw_data_fn', 'multiqc_{}'.format(table_id)) util_functions.write_data_file(dt.raw_vals, fn) report.saved_raw_data[fn] = dt.raw_vals return html
def make_table (dt): """ Build the HTML needed for a MultiQC table. :param data: MultiQC datatable object """ table_id = dt.pconfig.get('id', 'table_{}'.format(''.join(random.sample(letters, 4))) ) t_headers = OrderedDict() t_modal_headers = OrderedDict() t_rows = defaultdict(lambda: dict()) dt.raw_vals = defaultdict(lambda: dict()) empty_cells = dict() hidden_cols = 1 for idx, hs in enumerate(dt.headers): for k, header in hs.items(): rid = header['rid'] # Build the table header cell shared_key = '' if header.get('shared_key', None) is not None: shared_key = ' data-shared-key={}'.format(header['shared_key']) hide = '' muted = '' checked = ' checked="checked"' if header.get('hidden', False) is True: hide = 'hidden' muted = ' text-muted' checked = '' hidden_cols += 1 data_attr = 'data-chroma-scale="{}" data-chroma-max="{}" data-chroma-min="{}" {}' \ .format(header['scale'], header['dmax'], header['dmin'], shared_key) cell_contents = '<span data-toggle="tooltip" title="{}: {}">{}</span>' \ .format(header['namespace'], header['description'], header['title']) t_headers[rid] = '<th id="header_{rid}" class="chroma-col {rid} {h}" {d}>{c}</th>' \ .format(rid=rid, d=data_attr, h=hide, c=cell_contents) empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format(rid=rid, h=hide) # Build the modal table row t_modal_headers[rid] = """ <tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);"> <td class="sorthandle ui-sortable-handle">||</span></td> <td style="text-align:center;"> <input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}"> </td> <td>{name}</td> <td>{title}</td> <td>{desc}</td> <td>{col_id}</td> <td>{sk}</td> </tr>""".format( rid = rid, muted = muted, checked = checked, tid = table_id, col = header['colour'], name = header['namespace'], title = header['title'], desc = header['description'], col_id = '<code>{}</code>'.format(k), sk = header.get('shared_key', '') ) # Add the data table cells for (s_name, samp) in dt.data[idx].items(): if k in samp: val = samp[k] dt.raw_vals[s_name][rid] = val if 'modify' in header and callable(header['modify']): val = header['modify'](val) try: dmin = header['dmin'] dmax = header['dmax'] percentage = ((float(val) - dmin) / (dmax - dmin)) * 100; percentage = min(percentage, 100) percentage = max(percentage, 0) except (ZeroDivisionError,ValueError): percentage = 0 try: val = header['format'].format(val) except ValueError: try: val = header['format'].format(float(samp[k])) except ValueError: val = samp[k] except: val = samp[k] # Build HTML bar_html = '<span class="bar" style="width:{}%;"></span>'.format(percentage) val_html = '<span class="val">{}</span>'.format(val) wrapper_html = '<div class="wrapper">{}{}</div>'.format(bar_html, val_html) t_rows[s_name][rid] = \ '<td class="data-coloured {rid} {h}">{c}</td>'.format(rid=rid, h=hide, c=wrapper_html) # Remove header if we don't have any filled cells for it if sum([len(rows) for rows in t_rows.values()]) == 0: t_headers.pop(rid, None) t_modal_headers.pop(rid, None) logger.debug('Removing header {} from general stats table, as no data'.format(k)) # # Put everything together # # Buttons above the table html = """ <button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}"> <span class="glyphicon glyphicon-copy"></span> Copy table </button> <button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal"> <span class="glyphicon glyphicon-th"></span> Configure Columns </button> <button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;"> <span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight </button> <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small> """.format(tid=table_id, nrows=len(t_rows), ncols_vis = len(t_headers)-hidden_cols, ncols=len(t_headers)) # Build the table itself html += """ <div id="{tid}_container" class="mqc_table_container"> <div class="table-responsive"> <table id="{tid}" class="table table-condensed mqc_table"> """.format(tid=table_id) # Build the header row html += '<thead><tr><th class="rowheader">Sample Name</th>{}</tr></thead>'.format(''.join(t_headers.values())) # Build the table body html += '<tbody>' for s_name in sorted(t_rows.keys()): html += '<tr>' # Sample name row header html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format(sn=s_name) for k in t_headers: html += t_rows[s_name].get(k, empty_cells[k]) html += '</tr>' html += '</tbody></table></div></div>' # Build the bootstrap modal to customise columns and order html += """ <!-- MultiQC Table Columns Modal --> <div class="modal fade" id="{tid}_configModal" tabindex="-1"> <div class="modal-dialog modal-lg"> <div class="modal-content"> <div class="modal-header"> <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button> <h4 class="modal-title">{title}: Columns</h4> </div> <div class="modal-body"> <p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p> <p> <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button> <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button> </p> <table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table"> <thead> <tr> <th class="sorthandle" style="text-align:center;">Sort</th> <th style="text-align:center;">Visible</th> <th>Group</th> <th>Column</th> <th>Description</th> <th>ID</th> <th>Scale</th> </tr> </thead> <tbody> {trows} </tbody> </table> </div> <div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div> </div> </div> </div>""".format( tid=table_id, title=dt.pconfig.get('table_title', table_id), trows=''.join(t_modal_headers.values()) ) # Save the raw values to a file if requested if dt.pconfig.get('save_file') is True: fn = dt.pconfig.get('raw_data_fn', 'multiqc_{}'.format(table_id) ) util_functions.write_data_file(dt.raw_vals, fn ) return html
def write_data_file(self, data, fn, sort_cols=False, data_format=None): """ Saves raw data to a dictionary for downstream use, then redirects to report.write_data_file() to create the file in the report directory """ report.saved_raw_data[fn] = data util_functions.write_data_file(data, fn, sort_cols, data_format)