def __init__(self,
name='base',
anchor='base',
target=None,
href=None,
info=None,
extra=None):
# Custom options from user config that can overwrite module values
mod_cust_config = getattr(self, 'mod_cust_config', {})
self.name = mod_cust_config.get('name', name)
self.anchor = report.save_htmlid(mod_cust_config.get('anchor', anchor))
target = mod_cust_config.get('target', target)
href = mod_cust_config.get('href', href)
info = mod_cust_config.get('info', info)
extra = mod_cust_config.get('extra', extra)
if info is None:
info = ''
if extra is None:
extra = ''
if target is None:
target = self.name
if href is not None:
mname = '<a href="{}" target="_blank">{}</a>'.format(href, target)
else:
mname = target
self.intro = '<p>{} {}</p>{}'.format(mname, info, extra)
self.sections = list()
def add_section(self, name=None, anchor=None, description='', helptext='', plot='', content='', autoformat=True):
""" Add a section to the module report output """
# Default anchor
if anchor is None:
if name is not None:
nid = name.lower().strip().replace(' ','-')
anchor = '{}-{}'.format(self.anchor, nid)
else:
sl = len(self.sections) + 1
anchor = '{}-section-{}'.format(self.anchor, sl)
# Sanitise anchor ID and check for duplicates
anchor = report.save_htmlid(anchor)
# Format the content
if autoformat:
if len(description) > 0:
description = '<p class="mqc-section-description">{}</p>'.format(description)
if len(helptext) > 0:
helptext = '<p class="mqc-section-helptext">{}</p>'.format(helptext)
if len(plot) > 0:
plot = '<div class="mqc-section-plot">{}</div>'.format(plot)
self.sections.append({
'name': name,
'anchor': anchor,
'description': description,
'helptext': helptext,
'plot': plot,
'content': description + helptext + plot + content
})
def add_section(self, name=None, anchor=None, description='', comment='', helptext='', plot='', content='', autoformat=True, autoformat_type='markdown'):
""" Add a section to the module report output """
# Default anchor
if anchor is None:
if name is not None:
nid = name.lower().strip().replace(' ','-')
anchor = '{}-{}'.format(self.anchor, nid)
else:
sl = len(self.sections) + 1
anchor = '{}-section-{}'.format(self.anchor, sl)
# Append custom module anchor to the section if set
mod_cust_config = getattr(self, 'mod_cust_config', {})
if 'anchor' in mod_cust_config:
anchor = '{}_{}'.format(mod_cust_config['anchor'], anchor)
# Sanitise anchor ID and check for duplicates
anchor = report.save_htmlid(anchor)
# Skip if user has a config to remove this module section
if anchor in config.remove_sections:
logger.debug("Skipping section '{}' because specified in user config".format(anchor))
return
# See if we have a user comment in the config
if anchor in config.section_comments:
comment = config.section_comments[anchor]
# Format the content
if autoformat:
if len(description) > 0:
description = textwrap.dedent(description)
if autoformat_type == 'markdown':
description = markdown.markdown(description)
if len(comment) > 0:
comment = textwrap.dedent(comment)
if autoformat_type == 'markdown':
comment = markdown.markdown(comment)
if len(helptext) > 0:
helptext = textwrap.dedent(helptext)
if autoformat_type == 'markdown':
helptext = markdown.markdown(helptext)
# Strip excess whitespace
description = description.strip()
comment = comment.strip()
helptext = helptext.strip()
self.sections.append({
'name': name,
'anchor': anchor,
'description': description,
'comment': comment,
'helptext': helptext,
'plot': plot,
'content': content,
'print_section': any([ n is not None and len(n) > 0 for n in [description, comment, helptext, plot, content] ])
})
def make_plot(dt):
bs_id = dt.pconfig.get('id', 'table_{}'.format(''.join(random.sample(letters, 4))) )
# Sanitise plot ID and check for duplicates
bs_id = report.save_htmlid(bs_id)
categories = []
s_names = []
data = []
for idx, hs in enumerate(dt.headers):
for k, header in hs.items():
bcol = 'rgb({})'.format(header.get('colour', '204,204,204'))
categories.append({
'namespace': header['namespace'],
'title': header['title'],
'description': header['description'],
'max': header['dmax'],
'min': header['dmin'],
'suffix': header.get('suffix', ''),
'decimalPlaces': header.get('decimalPlaces', '2'),
'bordercol': bcol
});
# Add the data
thisdata = []
these_snames = []
for (s_name, samp) in dt.data[idx].items():
if k in samp:
val = samp[k]
if 'modify' in header and callable(header['modify']):
val = header['modify'](val)
thisdata.append(val)
these_snames.append(s_name)
data.append(thisdata)
s_names.append(these_snames)
# Plot HTML
html = """<div class="hc-plot-wrapper">
<div id="{bid}" class="hc-plot not_rendered hc-beeswarm-plot"><small>loading..</small></div>
</div>""".format(bid=bs_id)
report.num_hc_plots += 1
report.plot_data[bs_id] = {
'plot_type': 'beeswarm',
'samples': s_names,
'datasets': data,
'categories': categories
}
return html
def add_section(self,
name=None,
anchor=None,
description='',
helptext='',
plot='',
content='',
autoformat=True,
autoformat_type='markdown'):
""" Add a section to the module report output """
# Default anchor
if anchor is None:
if name is not None:
nid = name.lower().strip().replace(' ', '-')
anchor = '{}-{}'.format(self.anchor, nid)
else:
sl = len(self.sections) + 1
anchor = '{}-section-{}'.format(self.anchor, sl)
# Sanitise anchor ID and check for duplicates
anchor = report.save_htmlid(anchor)
# Format the content
if autoformat:
if len(description) > 0:
description = textwrap.dedent(description)
if autoformat_type == 'markdown':
description = markdown.markdown(description)
if len(helptext) > 0:
helptext = textwrap.dedent(helptext)
if autoformat_type == 'markdown':
helptext = markdown.markdown(helptext)
if len(plot) > 0:
plot = textwrap.dedent(plot)
if autoformat_type == 'markdown':
plot = markdown.markdown(plot)
self.sections.append({
'name':
name,
'anchor':
anchor,
'description':
description,
'helptext':
helptext,
'plot':
plot,
'content':
content,
'print_section':
any([
n is not None and len(n) > 0
for n in [description, helptext, plot, content]
])
})
def __init__(
self,
name="base",
anchor="base",
target=None,
href=None,
info=None,
comment=None,
extra=None,
autoformat=True,
autoformat_type="markdown",
):
# Custom options from user config that can overwrite base module values
mod_cust_config = getattr(self, "mod_cust_config", {})
self.name = mod_cust_config.get("name", name)
self.anchor = mod_cust_config.get("anchor", anchor)
target = mod_cust_config.get("target", target)
self.href = mod_cust_config.get("href", href)
self.info = mod_cust_config.get("info", info)
self.comment = mod_cust_config.get("comment", comment)
self.extra = mod_cust_config.get("extra", extra)
# Specific module level config to overwrite (e.g. config.bcftools, config.fastqc)
config.update({anchor: mod_cust_config.get("custom_config", {})})
# Sanitise anchor ID and check for duplicates
self.anchor = report.save_htmlid(self.anchor)
# See if we have a user comment in the config
if self.anchor in config.section_comments:
self.comment = config.section_comments[self.anchor]
if self.info is None:
self.info = ""
if self.extra is None:
self.extra = ""
if target is None:
target = self.name
if self.href is not None:
self.mname = '<a href="{}" target="_blank">{}</a>'.format(
self.href, target)
else:
self.mname = target
if self.href or self.info or self.extra:
self.intro = "<p>{} {}</p>{}".format(self.mname, self.info,
self.extra)
# Format the markdown strings
if autoformat:
if self.comment is not None:
self.comment = textwrap.dedent(self.comment)
if autoformat_type == "markdown":
self.comment = markdown.markdown(self.comment)
self.sections = list()
def add_section(self, name=None, anchor=None, description='', comment='', helptext='', plot='', content='', autoformat=True, autoformat_type='markdown'):
""" Add a section to the module report output """
# Default anchor
if anchor is None:
if name is not None:
nid = name.lower().strip().replace(' ','-')
anchor = '{}-{}'.format(self.anchor, nid)
else:
sl = len(self.sections) + 1
anchor = '{}-section-{}'.format(self.anchor, sl)
# Skip if user has a config to remove this module section
if anchor in config.remove_sections:
logger.debug("Skipping section '{}' because specified in user config".format(anchor))
return
# Sanitise anchor ID and check for duplicates
anchor = report.save_htmlid(anchor)
# See if we have a user comment in the config
if anchor in config.section_comments:
comment = config.section_comments[anchor]
# Format the content
if autoformat:
if len(description) > 0:
description = textwrap.dedent(description)
if autoformat_type == 'markdown':
description = markdown.markdown(description)
if len(comment) > 0:
comment = textwrap.dedent(comment)
if autoformat_type == 'markdown':
comment = markdown.markdown(comment)
if len(helptext) > 0:
helptext = textwrap.dedent(helptext)
if autoformat_type == 'markdown':
helptext = markdown.markdown(helptext)
# Strip excess whitespace
description = description.strip()
comment = comment.strip()
helptext = helptext.strip()
self.sections.append({
'name': name,
'anchor': anchor,
'description': description,
'comment': comment,
'helptext': helptext,
'plot': plot,
'content': content,
'print_section': any([ n is not None and len(n) > 0 for n in [description, comment, helptext, plot, content] ])
})
def highcharts_linegraph (plotdata, pconfig=None):
"""
Build the HTML needed for a HighCharts line graph. Should be
called by linegraph.plot(), which properly formats input data.
"""
if pconfig is None:
pconfig = {}
# Get the plot ID
if pconfig.get('id') is None:
pconfig['id'] = 'mqc_hcplot_'+''.join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig['id'] = report.save_htmlid(pconfig['id'])
# Build the HTML for the page
html = '<div class="mqc_hcplot_plotgroup">'
# Buttons to cycle through different datasets
if len(plotdata) > 1:
html += '<div class="btn-group hc_switch_group">\n'
for k, p in enumerate(plotdata):
active = 'active' if k == 0 else ''
try:
name = pconfig['data_labels'][k]['name']
except:
name = k+1
try:
ylab = 'data-ylab="{}"'.format(pconfig['data_labels'][k]['ylab'])
except:
ylab = 'data-ylab="{}"'.format(name) if name != k+1 else ''
try:
ymax = 'data-ymax="{}"'.format(pconfig['data_labels'][k]['ymax'])
except:
ymax = ''
try:
xlab = 'data-xlab="{}"'.format(pconfig['data_labels'][k]['xlab'])
except:
xlab = ''
html += '<button class="btn btn-default btn-sm {a}" data-action="set_data" {y} {ym} {x} data-newdata="{k}" data-target="{id}">{n}</button>\n'.format(a=active, id=pconfig['id'], n=name, y=ylab, ym=ymax, x=xlab, k=k)
html += '</div>\n\n'
# The plot div
html += '<div class="hc-plot-wrapper"><div id="{id}" class="hc-plot not_rendered hc-line-plot"><small>loading..</small></div></div></div> \n'.format(id=pconfig['id'])
report.num_hc_plots += 1
report.plot_data[pconfig['id']] = {
'plot_type': "xy_line",
'datasets': plotdata,
'config': pconfig
}
return html
def highcharts_scatter_plot(plotdata, pconfig=None):
"""
Build the HTML needed for a HighCharts scatter plot. Should be
called by scatter.plot(), which properly formats input data.
"""
if pconfig is None:
pconfig = {}
# Get the plot ID
if pconfig.get('id') is None:
pconfig['id'] = 'mqc_hcplot_' + ''.join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig['id'] = report.save_htmlid(pconfig['id'])
# Build the HTML for the page
html = '<div class="mqc_hcplot_plotgroup">'
# Buttons to cycle through different datasets
if len(plotdata) > 1:
html += '<div class="btn-group hc_switch_group">\n'
for k, p in enumerate(plotdata):
active = 'active' if k == 0 else ''
try:
name = pconfig['data_labels'][k]['name']
except:
name = k + 1
try:
ylab = 'data-ylab="{}"'.format(
pconfig['data_labels'][k]['ylab'])
except:
ylab = 'data-ylab="{}"'.format(name) if name != k + 1 else ''
try:
ymax = 'data-ymax="{}"'.format(
pconfig['data_labels'][k]['ymax'])
except:
ymax = ''
html += '<button class="btn btn-default btn-sm {a}" data-action="set_data" {y} {ym} data-newdata="{k}" data-target="{id}">{n}</button>\n'.format(
a=active, id=pconfig['id'], n=name, y=ylab, ym=ymax, k=k)
html += '</div>\n\n'
# The plot div
html += '<div class="hc-plot-wrapper"><div id="{id}" class="hc-plot not_rendered hc-scatter-plot"><small>loading..</small></div></div></div> \n'.format(
id=pconfig['id'])
report.num_hc_plots += 1
report.plot_data[pconfig['id']] = {
'plot_type': "scatter",
'datasets': plotdata,
'config': pconfig
}
return html
def __init__(self,
name='base',
anchor='base',
target=None,
href=None,
info=None,
comment=None,
extra=None,
autoformat=True,
autoformat_type='markdown'):
# Custom options from user config that can overwrite base module values
mod_cust_config = getattr(self, 'mod_cust_config', {})
self.name = mod_cust_config.get('name', name)
self.anchor = mod_cust_config.get('anchor', anchor)
target = mod_cust_config.get('target', target)
href = mod_cust_config.get('href', href)
info = mod_cust_config.get('info', info)
self.comment = mod_cust_config.get('comment', comment)
extra = mod_cust_config.get('extra', extra)
# Specific module level config to overwrite (e.g. config.bcftools, config.fastqc)
config.update({anchor: mod_cust_config.get('custom_config', {})})
# Sanitise anchor ID and check for duplicates
self.anchor = report.save_htmlid(self.anchor)
# See if we have a user comment in the config
if self.anchor in config.section_comments:
self.comment = config.section_comments[self.anchor]
if info is None:
info = ''
if extra is None:
extra = ''
if target is None:
target = self.name
if href is not None:
mname = '<a href="{}" target="_blank">{}</a>'.format(href, target)
else:
mname = target
if href or info or extra:
self.intro = '<p>{} {}</p>{}'.format(mname, info, extra)
# Format the markdown strings
if autoformat:
if self.comment is not None:
self.comment = textwrap.dedent(self.comment)
if autoformat_type == 'markdown':
self.comment = markdown.markdown(self.comment)
self.sections = list()
def highcharts_heatmap(data, xcats, ycats, pconfig=None):
"""
Build the HTML needed for a HighCharts line graph. Should be
called by plot_xy_data, which properly formats input data.
"""
if pconfig is None:
pconfig = {}
# Reformat the data for highcharts
pdata = []
for i, arr in enumerate(data):
for j, val in enumerate(arr):
pdata.append([j, i, val])
# Get the plot ID
if pconfig.get("id") is None:
pconfig["id"] = "mqc_hcplot_" + "".join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig["id"] = report.save_htmlid(pconfig["id"])
# Build the HTML for the page
html = '<div class="mqc_hcplot_plotgroup">'
# The 'sort by highlights button'
html += """<div class="btn-group hc_switch_group">
<button type="button" class="mqc_heatmap_sortHighlight btn btn-default btn-sm" data-target="#{id}" disabled="disabled">
<span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight
</button>
</div>""".format(id=pconfig["id"])
# The plot div
html += '<div class="hc-plot-wrapper"><div id="{id}" class="hc-plot not_rendered hc-heatmap"><small>loading..</small></div></div></div> \n'.format(
id=pconfig["id"])
report.num_hc_plots += 1
report.plot_data[pconfig["id"]] = {
"plot_type": "heatmap",
"data": pdata,
"xcats": xcats,
"ycats": ycats,
"config": pconfig,
}
return html
def highcharts_heatmap (data, xcats, ycats, pconfig=None):
"""
Build the HTML needed for a HighCharts line graph. Should be
called by plot_xy_data, which properly formats input data.
"""
if pconfig is None:
pconfig = {}
# Reformat the data for highcharts
pdata = []
for i, arr in enumerate(data):
for j, val in enumerate(arr):
pdata.append([j,i,val])
# Get the plot ID
if pconfig.get('id') is None:
pconfig['id'] = 'mqc_hcplot_'+''.join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig['id'] = report.save_htmlid(pconfig['id'])
# Build the HTML for the page
html = '<div class="mqc_hcplot_plotgroup">'
# The 'sort by highlights button'
html += """<div class="btn-group hc_switch_group">
<button type="button" class="mqc_heatmap_sortHighlight btn btn-default btn-sm" data-target="#{id}" disabled="disabled">
<span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight
</button>
</div>""".format(id=pconfig['id'])
# The plot div
html += '<div class="hc-plot-wrapper"><div id="{id}" class="hc-plot not_rendered hc-heatmap"><small>loading..</small></div></div></div> \n'.format(id=pconfig['id'])
report.num_hc_plots += 1
report.plot_data[pconfig['id']] = {
'plot_type': 'heatmap',
'data': pdata,
'xcats': xcats,
'ycats': ycats,
'config': pconfig
}
return html
def __init__(self, name='base', anchor='base', target=None, href=None, info=None, comment=None, extra=None,
autoformat=True, autoformat_type='markdown'):
# Custom options from user config that can overwrite base module values
mod_cust_config = getattr(self, 'mod_cust_config', {})
self.name = mod_cust_config.get('name', name)
self.anchor = report.save_htmlid( mod_cust_config.get('anchor', anchor) )
target = mod_cust_config.get('target', target)
href = mod_cust_config.get('href', href)
info = mod_cust_config.get('info', info)
self.comment = mod_cust_config.get('comment', comment)
extra = mod_cust_config.get('extra', extra)
# Specific module level config to overwrite (e.g. config.bcftools, config.fastqc)
config.update({anchor: mod_cust_config.get('custom_config', {})})
# See if we have a user comment in the config
if self.anchor in config.section_comments:
self.comment = config.section_comments[self.anchor]
if info is None:
info = ''
if extra is None:
extra = ''
if target is None:
target = self.name
if href is not None:
mname = '<a href="{}" target="_blank">{}</a>'.format(href, target)
else:
mname = target
self.intro = '<p>{} {}</p>{}'.format( mname, info, extra )
# Format the markdown strings
if autoformat:
if self.comment is not None:
self.comment = textwrap.dedent(self.comment)
if autoformat_type == 'markdown':
self.comment = markdown.markdown(self.comment)
self.sections = list()
def export_scatter(plotdata, pconfig):
# for fformat in config.export_plot_formats: # Only need png for now
nb_plots = len(plotdata)
nb_rows = ceil((nb_plots * 1.0) / 2.0)
vert_size = 7 * nb_rows
plt.axis('equal')
p = plt.figure(figsize=(14, vert_size), frameon=False)
num_pat = re.compile('[0-9]+')
for k, single_pd in enumerate(plotdata):
x = [point['x'] for point in single_pd]
y = [point['y'] for point in single_pd]
color = [
re.findall(num_pat, point.get('color', 'rgb(244, 91, 91, 1)'))
for point in single_pd
]
color = np.array(color).astype(int)[:, 3]
axes = p.add_subplot(nb_rows, 2, k + 1)
# _ = [axes.scatter(xc,yc,c=cc, s=2.5) for xc,yc,cc in zip(x,y,color)]
axes.scatter(x, y, c=color / 255.0, s=2.5)
axes.tick_params(labelsize=8,
direction='out',
left=False,
right=False,
top=False,
bottom=False)
axes.set_xlabel(pconfig['data_labels'][k].get('xlab', ''))
axes.set_ylabel(pconfig['data_labels'][k].get('ylab', ''))
# define plot id
# name = pconfig['data_labels'][k].get('name','')
# pid = 'mqc_{}_{}'.format(pconfig['id'], name)
pid = 'mqc_{}'.format(pconfig['id'])
pid = report.save_htmlid(pid, skiplint=True)
# Make the directory if it doesn't already exist
plot_dir = os.path.join(config.plots_dir, 'png')
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
# Make and save plot
plot_fn = os.path.join(plot_dir, '{}.{}'.format(pid, 'png'))
p.savefig(plot_fn, format='png')
def make_table (dt):
"""
Build the HTML needed for a MultiQC table.
:param data: MultiQC datatable object
"""
table_id = dt.pconfig.get('id', 'table_{}'.format(''.join(random.sample(letters, 4))) )
table_id = report.save_htmlid(table_id)
t_headers = OrderedDict()
t_modal_headers = OrderedDict()
t_rows = OrderedDict()
dt.raw_vals = defaultdict(lambda: dict())
empty_cells = dict()
hidden_cols = 1
table_title = dt.pconfig.get('table_title')
if table_title is None:
table_title = table_id.replace("_", " ").title()
for idx, k, header in dt.get_headers_in_order():
rid = header['rid']
# Build the table header cell
shared_key = ''
if header.get('shared_key', None) is not None:
shared_key = ' data-shared-key={}'.format(header['shared_key'])
hide = ''
muted = ''
checked = ' checked="checked"'
if header.get('hidden', False) is True:
hide = 'hidden'
muted = ' text-muted'
checked = ''
hidden_cols += 1
data_attr = 'data-dmax="{}" data-dmin="{}" data-namespace="{}" {}' \
.format(header['dmax'], header['dmin'], header['namespace'], shared_key)
cell_contents = '<span class="mqc_table_tooltip" title="{}: {}">{}</span>' \
.format(header['namespace'], header['description'], header['title'])
t_headers[rid] = '<th id="header_{rid}" class="{rid} {h}" {da}>{c}</th>' \
.format(rid=rid, h=hide, da=data_attr, c=cell_contents)
empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format(rid=rid, h=hide)
# Build the modal table row
t_modal_headers[rid] = """
<tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);">
<td class="sorthandle ui-sortable-handle">||</span></td>
<td style="text-align:center;">
<input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}">
</td>
<td>{name}</td>
<td>{title}</td>
<td>{desc}</td>
<td>{col_id}</td>
<td>{sk}</td>
</tr>""".format(
rid = rid,
muted = muted,
checked = checked,
tid = table_id,
col = header['colour'],
name = header['namespace'],
title = header['title'],
desc = header['description'],
col_id = '<code>{}</code>'.format(k),
sk = header.get('shared_key', '')
)
# Make a colour scale
if header['scale'] == False:
c_scale = None
else:
c_scale = mqc_colour.mqc_colour_scale(header['scale'], header['dmin'], header['dmax'])
# Add the data table cells
for (s_name, samp) in dt.data[idx].items():
if k in samp:
val = samp[k]
kname = '{}_{}'.format(header['namespace'], rid)
dt.raw_vals[s_name][kname] = val
if 'modify' in header and callable(header['modify']):
val = header['modify'](val)
try:
dmin = header['dmin']
dmax = header['dmax']
percentage = ((float(val) - dmin) / (dmax - dmin)) * 100
percentage = min(percentage, 100)
percentage = max(percentage, 0)
except (ZeroDivisionError,ValueError):
percentage = 0
try:
valstring = str(header['format'].format(val))
except ValueError:
try:
valstring = str(header['format'].format(float(val)))
except ValueError:
valstring = str(val)
except:
valstring = str(val)
# This is horrible, but Python locale settings are worse
if config.thousandsSep_format is None:
config.thousandsSep_format = '<span class="mqc_thousandSep"></span>'
if config.decimalPoint_format is None:
config.decimalPoint_format = '.'
valstring = valstring.replace('.', 'DECIMAL').replace(',', 'THOUSAND')
valstring = valstring.replace('DECIMAL', config.decimalPoint_format).replace('THOUSAND', config.thousandsSep_format)
# Percentage suffixes etc
valstring += header.get('suffix', '')
# Conditional formatting
cmatches = { cfck: False for cfc in config.table_cond_formatting_colours for cfck in cfc }
# Find general rules followed by column-specific rules
for cfk in ['all_columns', rid]:
if cfk in config.table_cond_formatting_rules:
# Loop through match types
for ftype in cmatches.keys():
# Loop through array of comparison types
for cmp in config.table_cond_formatting_rules[cfk].get(ftype, []):
try:
# Each comparison should be a dict with single key: val
if 's_eq' in cmp and str(cmp['s_eq']).lower() == str(val).lower():
cmatches[ftype] = True
if 's_contains' in cmp and str(cmp['s_contains']).lower() in str(val).lower():
cmatches[ftype] = True
if 's_ne' in cmp and str(cmp['s_ne']).lower() != str(val).lower():
cmatches[ftype] = True
if 'eq' in cmp and float(cmp['eq']) == float(val):
cmatches[ftype] = True
if 'ne' in cmp and float(cmp['ne']) != float(val):
cmatches[ftype] = True
if 'gt' in cmp and float(cmp['gt']) < float(val):
cmatches[ftype] = True
if 'lt' in cmp and float(cmp['lt']) > float(val):
cmatches[ftype] = True
except:
logger.warn("Not able to apply table conditional formatting to '{}' ({})".format(val, cmp))
# Apply HTML in order of config keys
bgcol = None
for cfc in config.table_cond_formatting_colours:
for cfck in cfc: # should always be one, but you never know
if cmatches[cfck]:
bgcol = cfc[cfck]
if bgcol is not None:
valstring = '<span class="badge" style="background-color:{}">{}</span>'.format(bgcol, valstring)
# Build HTML
if not header['scale']:
if s_name not in t_rows:
t_rows[s_name] = dict()
t_rows[s_name][rid] = '<td class="{rid} {h}">{v}</td>'.format(rid=rid, h=hide, v=valstring)
else:
if c_scale is not None:
col = ' background-color:{};'.format(c_scale.get_colour(val))
else:
col = ''
bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(percentage, col)
val_html = '<span class="val">{}</span>'.format(valstring)
wrapper_html = '<div class="wrapper">{}{}</div>'.format(bar_html, val_html)
if s_name not in t_rows:
t_rows[s_name] = dict()
t_rows[s_name][rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(rid=rid, h=hide, c=wrapper_html)
# Remove header if we don't have any filled cells for it
if sum([len(rows) for rows in t_rows.values()]) == 0:
t_headers.pop(rid, None)
t_modal_headers.pop(rid, None)
logger.debug('Removing header {} from general stats table, as no data'.format(k))
#
# Put everything together
#
# Buttons above the table
html = ''
if not config.simple_output:
# Copy Table Button
html += """
<button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}">
<span class="glyphicon glyphicon-copy"></span> Copy table
</button>
""".format(tid=table_id)
# Configure Columns Button
if len(t_headers) > 1:
html += """
<button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal">
<span class="glyphicon glyphicon-th"></span> Configure Columns
</button>
""".format(tid=table_id)
# Sort By Highlight button
html += """
<button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;">
<span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight
</button>
""".format(tid=table_id)
# Scatter Plot Button
if len(t_headers) > 1:
html += """
<button type="button" class="mqc_table_makeScatter btn btn-default btn-sm" data-toggle="modal" data-target="#tableScatterModal" data-table="#{tid}">
<span class="glyphicon glyphicon glyphicon-stats"></span> Plot
</button>
""".format(tid=table_id)
# "Showing x of y columns" text
html += """
<small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small>
""".format(tid=table_id, nrows=len(t_rows), ncols_vis = (len(t_headers)+1)-hidden_cols, ncols=len(t_headers))
# Build the table itself
collapse_class = 'mqc-table-collapse' if len(t_rows) > 10 and config.collapse_tables else ''
html += """
<div id="{tid}_container" class="mqc_table_container">
<div class="table-responsive mqc-table-responsive {cc}">
<table id="{tid}" class="table table-condensed mqc_table" data-title="{title}">
""".format( tid=table_id, title=table_title, cc=collapse_class)
# Build the header row
col1_header = dt.pconfig.get('col1_header', 'Sample Name')
html += '<thead><tr><th class="rowheader">{}</th>{}</tr></thead>'.format(col1_header, ''.join(t_headers.values()))
# Build the table body
html += '<tbody>'
t_row_keys = t_rows.keys()
if dt.pconfig.get('sortRows') is not False:
t_row_keys = sorted(t_row_keys)
for s_name in t_row_keys:
html += '<tr>'
# Sample name row header
html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format(sn=s_name)
for k in t_headers:
html += t_rows[s_name].get(k, empty_cells[k])
html += '</tr>'
html += '</tbody></table></div>'
if len(t_rows) > 10 and config.collapse_tables:
html += '<div class="mqc-table-expand"><span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span></div>'
html += '</div>'
# Build the bootstrap modal to customise columns and order
if not config.simple_output:
html += """
<!-- MultiQC Table Columns Modal -->
<div class="modal fade" id="{tid}_configModal" tabindex="-1">
<div class="modal-dialog modal-lg">
<div class="modal-content">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button>
<h4 class="modal-title">{title}: Columns</h4>
</div>
<div class="modal-body">
<p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p>
<p>
<button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button>
<button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button>
</p>
<table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table" data-title="{title}">
<thead>
<tr>
<th class="sorthandle" style="text-align:center;">Sort</th>
<th style="text-align:center;">Visible</th>
<th>Group</th>
<th>Column</th>
<th>Description</th>
<th>ID</th>
<th>Scale</th>
</tr>
</thead>
<tbody>
{trows}
</tbody>
</table>
</div>
<div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div>
</div> </div> </div>""".format( tid=table_id, title=table_title, trows=''.join(t_modal_headers.values()) )
# Save the raw values to a file if requested
if dt.pconfig.get('save_file') is True:
fn = dt.pconfig.get('raw_data_fn', 'multiqc_{}'.format(table_id) )
util_functions.write_data_file(dt.raw_vals, fn )
report.saved_raw_data[fn] = dt.raw_vals
return html
def __init__(self, data, headers=None, pconfig=None):
"""Prepare data for use in a table or plot"""
if headers is None:
headers = []
if pconfig is None:
pconfig = {}
# Given one dataset - turn it into a list
if type(data) is not list:
data = [data]
if type(headers) is not list:
headers = [headers]
sectcols = [
"55,126,184",
"77,175,74",
"152,78,163",
"255,127,0",
"228,26,28",
"255,255,51",
"166,86,40",
"247,129,191",
"153,153,153",
]
shared_keys = defaultdict(lambda: dict())
# Go through each table section
for idx, d in enumerate(data):
# Get the header keys
try:
keys = headers[idx].keys()
assert len(keys) > 0
except (IndexError, AttributeError, AssertionError):
pconfig["only_defined_headers"] = False
# Add header keys from the data
if pconfig.get("only_defined_headers", True) is False:
# Get the keys from the data
keys = list()
for samp in d.values():
for k in samp.keys():
if k not in keys:
keys.append(k)
# If we don't have a headers dict for this data set yet, create one
try:
headers[idx]
except IndexError:
headers.append(list)
headers[idx] = OrderedDict()
else:
# Convert the existing headers into an OrderedDict (eg. if parsed from a config)
od_tuples = [(key, headers[idx][key]) for key in headers[idx].keys()]
headers[idx] = OrderedDict(od_tuples)
# Create empty header configs for each new data key
for k in keys:
if k not in headers[idx]:
headers[idx][k] = {}
# Ensure that keys are strings, not numeric
keys = [str(k) for k in keys]
for k in list(headers[idx].keys()):
headers[idx][str(k)] = headers[idx].pop(k)
# Ensure that all sample names are strings as well
cdata = OrderedDict()
for k, v in data[idx].items():
cdata[str(k)] = v
data[idx] = cdata
for s_name in data[idx].keys():
for k in list(data[idx][s_name].keys()):
data[idx][s_name][str(k)] = data[idx][s_name].pop(k)
# Check that we have some data in each column
empties = list()
for k in keys:
n = 0
for samp in d.values():
if k in samp:
n += 1
if n == 0:
empties.append(k)
for k in empties:
keys = [j for j in keys if j != k]
del headers[idx][k]
for k in keys:
# Unique id to avoid overwriting by other datasets
if "rid" not in headers[idx][k]:
headers[idx][k]["rid"] = report.save_htmlid(re.sub(r"\W+", "_", k).strip().strip("_"))
# Applying defaults presets for data keys if shared_key is set to base_count or read_count
shared_key = headers[idx][k].get("shared_key", None)
if shared_key in ["read_count", "base_count"]:
if shared_key == "read_count":
multiplier = config.read_count_multiplier
else:
multiplier = config.base_count_multiplier
if headers[idx][k].get("modify") is None:
headers[idx][k]["modify"] = lambda x: x * multiplier
if headers[idx][k].get("min") is None:
headers[idx][k]["min"] = 0
if headers[idx][k].get("format") is None:
if multiplier == 1:
headers[idx][k]["format"] = "{:,.0f}"
# Use defaults / data keys if headers not given
headers[idx][k]["namespace"] = headers[idx][k].get("namespace", pconfig.get("namespace", ""))
headers[idx][k]["title"] = headers[idx][k].get("title", k)
headers[idx][k]["description"] = headers[idx][k].get("description", headers[idx][k]["title"])
headers[idx][k]["scale"] = headers[idx][k].get("scale", pconfig.get("scale", "GnBu"))
headers[idx][k]["format"] = headers[idx][k].get("format", pconfig.get("format", "{:,.1f}"))
headers[idx][k]["colour"] = headers[idx][k].get("colour", pconfig.get("colour", None))
headers[idx][k]["hidden"] = headers[idx][k].get("hidden", pconfig.get("hidden", None))
headers[idx][k]["max"] = headers[idx][k].get("max", pconfig.get("max", None))
headers[idx][k]["min"] = headers[idx][k].get("min", pconfig.get("min", None))
headers[idx][k]["ceiling"] = headers[idx][k].get("ceiling", pconfig.get("ceiling", None))
headers[idx][k]["floor"] = headers[idx][k].get("floor", pconfig.get("floor", None))
headers[idx][k]["minRange"] = headers[idx][k].get("minRange", pconfig.get("minRange", None))
headers[idx][k]["shared_key"] = headers[idx][k].get("shared_key", pconfig.get("shared_key", None))
headers[idx][k]["modify"] = headers[idx][k].get("modify", pconfig.get("modify", None))
headers[idx][k]["placement"] = float(headers[idx][k].get("placement", 1000))
if headers[idx][k]["colour"] is None:
cidx = idx
while cidx >= len(sectcols):
cidx -= len(sectcols)
headers[idx][k]["colour"] = sectcols[cidx]
# Overwrite hidden if set in user config
for ns in config.table_columns_visible.keys():
# Make namespace key case insensitive
if ns.lower() == headers[idx][k]["namespace"].lower():
# First - if config value is a bool, set all module columns to that value
if isinstance(config.table_columns_visible[ns], bool):
headers[idx][k]["hidden"] = not config.table_columns_visible[ns]
# Not a bool, assume a dict of the specific column IDs
else:
try:
# Config has True = visibile, False = Hidden. Here we're setting "hidden" which is inverse
headers[idx][k]["hidden"] = not config.table_columns_visible[ns][k]
except KeyError:
pass
# Also overwite placement if set in config
try:
headers[idx][k]["placement"] = float(
config.table_columns_placement[headers[idx][k]["namespace"]][k]
)
except (KeyError, ValueError):
try:
headers[idx][k]["placement"] = float(config.table_columns_placement[pconfig["id"]][k])
except (KeyError, ValueError):
pass
# Work out max and min value if not given
setdmax = False
setdmin = False
try:
headers[idx][k]["dmax"] = float(headers[idx][k]["max"])
except TypeError:
headers[idx][k]["dmax"] = 0
setdmax = True
try:
headers[idx][k]["dmin"] = float(headers[idx][k]["min"])
except TypeError:
headers[idx][k]["dmin"] = 0
setdmin = True
# Figure out the min / max if not supplied
if setdmax or setdmin:
for s_name, samp in data[idx].items():
try:
val = float(samp[k])
if callable(headers[idx][k]["modify"]):
val = float(headers[idx][k]["modify"](val))
if setdmax:
headers[idx][k]["dmax"] = max(headers[idx][k]["dmax"], val)
if setdmin:
headers[idx][k]["dmin"] = min(headers[idx][k]["dmin"], val)
except (ValueError, TypeError):
val = samp[k] # couldn't convert to float - keep as a string
except KeyError:
pass # missing data - skip
# Limit auto-generated scales with floor, ceiling and minRange.
if headers[idx][k]["ceiling"] is not None and headers[idx][k]["max"] is None:
headers[idx][k]["dmax"] = min(headers[idx][k]["dmax"], float(headers[idx][k]["ceiling"]))
if headers[idx][k]["floor"] is not None and headers[idx][k]["min"] is None:
headers[idx][k]["dmin"] = max(headers[idx][k]["dmin"], float(headers[idx][k]["floor"]))
if headers[idx][k]["minRange"] is not None:
drange = headers[idx][k]["dmax"] - headers[idx][k]["dmin"]
if drange < float(headers[idx][k]["minRange"]):
headers[idx][k]["dmax"] = headers[idx][k]["dmin"] + float(headers[idx][k]["minRange"])
# Collect settings for shared keys
shared_keys = defaultdict(lambda: dict())
for idx, hs in enumerate(headers):
for k in hs.keys():
sk = headers[idx][k]["shared_key"]
if sk is not None:
shared_keys[sk]["dmax"] = max(
headers[idx][k]["dmax"], shared_keys[sk].get("dmax", headers[idx][k]["dmax"])
)
shared_keys[sk]["dmin"] = min(
headers[idx][k]["dmin"], shared_keys[sk].get("dmin", headers[idx][k]["dmin"])
)
# Overwrite shared key settings and at the same time assign to buckets for sorting
# Within each section of headers, sort explicitly by 'title' if the dict
# is not already ordered, so the final ordering is by:
# placement > section > explicit_ordering > title
# Of course, the user can shuffle these manually.
self.headers_in_order = defaultdict(list)
for idx, hs in enumerate(headers):
keys_in_section = hs.keys()
if type(hs) is not OrderedDict:
keys_in_section = sorted(keys_in_section, key=lambda k: headers[idx][k]["title"])
for k in keys_in_section:
sk = headers[idx][k]["shared_key"]
if sk is not None:
headers[idx][k]["dmax"] = shared_keys[sk]["dmax"]
headers[idx][k]["dmin"] = shared_keys[sk]["dmin"]
self.headers_in_order[headers[idx][k]["placement"]].append((idx, k))
# Skip any data that is not used in the table
# Would be ignored for making the table anyway, but can affect whether a beeswarm plot is used
for idx, d in enumerate(data):
for s_name in list(d.keys()):
if not any(h in data[idx][s_name].keys() for h in headers[idx]):
del data[idx][s_name]
# Assign to class
self.data = data
self.headers = headers
self.pconfig = pconfig
def add_section(
self,
name=None,
anchor=None,
description="",
comment="",
helptext="",
plot="",
content="",
autoformat=True,
autoformat_type="markdown",
):
"""Add a section to the module report output"""
# Default anchor
if anchor is None:
if name is not None:
nid = name.lower().strip().replace(" ", "-")
anchor = "{}-{}".format(self.anchor, nid)
else:
sl = len(self.sections) + 1
anchor = "{}-section-{}".format(self.anchor, sl)
# Append custom module anchor to the section if set
mod_cust_config = getattr(self, "mod_cust_config", {})
if "anchor" in mod_cust_config:
anchor = "{}_{}".format(mod_cust_config["anchor"], anchor)
# Sanitise anchor ID and check for duplicates
anchor = report.save_htmlid(anchor)
# Skip if user has a config to remove this module section
if anchor in config.remove_sections:
logger.debug(
"Skipping section '{}' because specified in user config".
format(anchor))
return
# See if we have a user comment in the config
if anchor in config.section_comments:
comment = config.section_comments[anchor]
# Format the content
if autoformat:
if len(description) > 0:
description = textwrap.dedent(description)
if autoformat_type == "markdown":
description = markdown.markdown(description)
if len(comment) > 0:
comment = textwrap.dedent(comment)
if autoformat_type == "markdown":
comment = markdown.markdown(comment)
if len(helptext) > 0:
helptext = textwrap.dedent(helptext)
if autoformat_type == "markdown":
helptext = markdown.markdown(helptext)
# Strip excess whitespace
description = description.strip()
comment = comment.strip()
helptext = helptext.strip()
self.sections.append({
"name":
name,
"anchor":
anchor,
"description":
description,
"comment":
comment,
"helptext":
helptext,
"plot":
plot,
"content":
content,
"print_section":
any([
n is not None and len(n) > 0
for n in [description, comment, helptext, plot, content]
]),
})
def make_table(dt):
"""
Build the HTML needed for a MultiQC table.
:param data: MultiQC datatable object
"""
table_id = dt.pconfig.get(
"id", "table_{}".format("".join(random.sample(letters, 4))))
table_id = report.save_htmlid(table_id)
t_headers = OrderedDict()
t_modal_headers = OrderedDict()
t_rows = OrderedDict()
t_rows_empty = OrderedDict()
dt.raw_vals = defaultdict(lambda: dict())
empty_cells = dict()
hidden_cols = 1
table_title = dt.pconfig.get("table_title")
if table_title is None:
table_title = table_id.replace("_", " ").title()
for idx, k, header in dt.get_headers_in_order():
rid = header["rid"]
# Build the table header cell
shared_key = ""
if header.get("shared_key", None) is not None:
shared_key = " data-shared-key={}".format(header["shared_key"])
hide = ""
muted = ""
checked = ' checked="checked"'
if header.get("hidden", False) is True:
hide = "hidden"
muted = " text-muted"
checked = ""
hidden_cols += 1
data_attr = 'data-dmax="{}" data-dmin="{}" data-namespace="{}" {}'.format(
header["dmax"], header["dmin"], header["namespace"], shared_key)
cell_contents = '<span class="mqc_table_tooltip" title="{}: {}">{}</span>'.format(
header["namespace"], header["description"], header["title"])
t_headers[
rid] = '<th id="header_{rid}" class="{rid} {h}" {da}>{c}</th>'.format(
rid=rid, h=hide, da=data_attr, c=cell_contents)
empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format(
rid=rid, h=hide)
# Build the modal table row
t_modal_headers[rid] = """
<tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);">
<td class="sorthandle ui-sortable-handle">||</span></td>
<td style="text-align:center;">
<input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}">
</td>
<td>{name}</td>
<td>{title}</td>
<td>{desc}</td>
<td>{col_id}</td>
<td>{sk}</td>
</tr>""".format(
rid=rid,
muted=muted,
checked=checked,
tid=table_id,
col=header["colour"],
name=header["namespace"],
title=header["title"],
desc=header["description"],
col_id="<code>{}</code>".format(k),
sk=header.get("shared_key", ""),
)
# Make a colour scale
if header["scale"] == False:
c_scale = None
else:
c_scale = mqc_colour.mqc_colour_scale(header["scale"],
header["dmin"],
header["dmax"])
# Add the data table cells
for (s_name, samp) in dt.data[idx].items():
if k in samp:
val = samp[k]
kname = "{}_{}".format(header["namespace"], rid)
dt.raw_vals[s_name][kname] = val
if "modify" in header and callable(header["modify"]):
val = header["modify"](val)
try:
dmin = header["dmin"]
dmax = header["dmax"]
percentage = ((float(val) - dmin) / (dmax - dmin)) * 100
percentage = min(percentage, 100)
percentage = max(percentage, 0)
except (ZeroDivisionError, ValueError):
percentage = 0
try:
valstring = str(header["format"].format(val))
except ValueError:
try:
valstring = str(header["format"].format(float(val)))
except ValueError:
valstring = str(val)
except:
valstring = str(val)
# This is horrible, but Python locale settings are worse
if config.thousandsSep_format is None:
config.thousandsSep_format = '<span class="mqc_thousandSep"></span>'
if config.decimalPoint_format is None:
config.decimalPoint_format = "."
valstring = valstring.replace(".", "DECIMAL").replace(
",", "THOUSAND")
valstring = valstring.replace(
"DECIMAL", config.decimalPoint_format).replace(
"THOUSAND", config.thousandsSep_format)
# Percentage suffixes etc
valstring += header.get("suffix", "")
# Conditional formatting
cmatches = {
cfck: False
for cfc in config.table_cond_formatting_colours
for cfck in cfc
}
# Find general rules followed by column-specific rules
for cfk in ["all_columns", rid]:
if cfk in config.table_cond_formatting_rules:
# Loop through match types
for ftype in cmatches.keys():
# Loop through array of comparison types
for cmp in config.table_cond_formatting_rules[
cfk].get(ftype, []):
try:
# Each comparison should be a dict with single key: val
if "s_eq" in cmp and str(
cmp["s_eq"]).lower() == str(
val).lower():
cmatches[ftype] = True
if "s_contains" in cmp and str(
cmp["s_contains"]).lower() in str(
val).lower():
cmatches[ftype] = True
if "s_ne" in cmp and str(
cmp["s_ne"]).lower() != str(
val).lower():
cmatches[ftype] = True
if "eq" in cmp and float(
cmp["eq"]) == float(val):
cmatches[ftype] = True
if "ne" in cmp and float(
cmp["ne"]) != float(val):
cmatches[ftype] = True
if "gt" in cmp and float(
cmp["gt"]) < float(val):
cmatches[ftype] = True
if "lt" in cmp and float(
cmp["lt"]) > float(val):
cmatches[ftype] = True
except:
logger.warning(
"Not able to apply table conditional formatting to '{}' ({})"
.format(val, cmp))
# Apply HTML in order of config keys
bgcol = None
for cfc in config.table_cond_formatting_colours:
for cfck in cfc: # should always be one, but you never know
if cmatches[cfck]:
bgcol = cfc[cfck]
if bgcol is not None:
valstring = '<span class="badge" style="background-color:{}">{}</span>'.format(
bgcol, valstring)
# Build HTML
if not header["scale"]:
if s_name not in t_rows:
t_rows[s_name] = dict()
t_rows[s_name][
rid] = '<td class="{rid} {h}">{v}</td>'.format(
rid=rid, h=hide, v=valstring)
else:
if c_scale is not None:
col = " background-color:{};".format(
c_scale.get_colour(val))
else:
col = ""
bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(
percentage, col)
val_html = '<span class="val">{}</span>'.format(valstring)
wrapper_html = '<div class="wrapper">{}{}</div>'.format(
bar_html, val_html)
if s_name not in t_rows:
t_rows[s_name] = dict()
t_rows[s_name][
rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(
rid=rid, h=hide, c=wrapper_html)
# Is this cell hidden or empty?
if s_name not in t_rows_empty:
t_rows_empty[s_name] = dict()
t_rows_empty[s_name][rid] = header.get(
"hidden", False) or str(val).strip() == ""
# Remove header if we don't have any filled cells for it
if sum([len(rows) for rows in t_rows.values()]) == 0:
if header.get("hidden", False) is True:
hidden_cols -= 1
t_headers.pop(rid, None)
t_modal_headers.pop(rid, None)
logger.debug("Removing header {} from table, as no data".format(k))
#
# Put everything together
#
# Buttons above the table
html = ""
if not config.simple_output:
# Copy Table Button
html += """
<button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}">
<span class="glyphicon glyphicon-copy"></span> Copy table
</button>
""".format(tid=table_id)
# Configure Columns Button
if len(t_headers) > 1:
html += """
<button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal">
<span class="glyphicon glyphicon-th"></span> Configure Columns
</button>
""".format(tid=table_id)
# Sort By Highlight button
html += """
<button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;">
<span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight
</button>
""".format(tid=table_id)
# Scatter Plot Button
if len(t_headers) > 1:
html += """
<button type="button" class="mqc_table_makeScatter btn btn-default btn-sm" data-toggle="modal" data-target="#tableScatterModal" data-table="#{tid}">
<span class="glyphicon glyphicon glyphicon-stats"></span> Plot
</button>
""".format(tid=table_id)
# "Showing x of y columns" text
row_visibilities = [
all(t_rows_empty[s_name].values()) for s_name in t_rows_empty
]
visible_rows = [x for x in row_visibilities if not x]
# Visible rows
t_showing_rows_txt = (
'Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nvisrows}</sup>/<sub>{nrows}</sub> rows'
.format(tid=table_id,
nvisrows=len(visible_rows),
nrows=len(t_rows)))
# How many columns are visible?
ncols_vis = (len(t_headers) + 1) - hidden_cols
t_showing_cols_txt = ""
if len(t_headers) > 1:
t_showing_cols_txt = ' and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns'.format(
tid=table_id, ncols_vis=ncols_vis, ncols=len(t_headers))
# Build table header text
html += """
<small id="{tid}_numrows_text" class="mqc_table_numrows_text">{rows}{cols}.</small>
""".format(tid=table_id,
rows=t_showing_rows_txt,
cols=t_showing_cols_txt)
# Build the table itself
collapse_class = "mqc-table-collapse" if len(
t_rows) > 10 and config.collapse_tables else ""
html += """
<div id="{tid}_container" class="mqc_table_container">
<div class="table-responsive mqc-table-responsive {cc}">
<table id="{tid}" class="table table-condensed mqc_table" data-title="{title}">
""".format(tid=table_id, title=table_title, cc=collapse_class)
# Build the header row
col1_header = dt.pconfig.get("col1_header", "Sample Name")
html += '<thead><tr><th class="rowheader">{}</th>{}</tr></thead>'.format(
col1_header, "".join(t_headers.values()))
# Build the table body
html += "<tbody>"
t_row_keys = t_rows.keys()
if dt.pconfig.get("sortRows") is not False:
t_row_keys = sorted(t_row_keys)
for s_name in t_row_keys:
# Hide the row if all cells are empty or hidden
row_hidden = ' style="display:none"' if all(
t_rows_empty[s_name].values()) else ""
html += "<tr{}>".format(row_hidden)
# Sample name row header
html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format(
sn=s_name)
for k in t_headers:
html += t_rows[s_name].get(k, empty_cells[k])
html += "</tr>"
html += "</tbody></table></div>"
if len(t_rows) > 10 and config.collapse_tables:
html += '<div class="mqc-table-expand"><span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span></div>'
html += "</div>"
# Build the bootstrap modal to customise columns and order
if not config.simple_output:
html += """
<!-- MultiQC Table Columns Modal -->
<div class="modal fade" id="{tid}_configModal" tabindex="-1">
<div class="modal-dialog modal-lg">
<div class="modal-content">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button>
<h4 class="modal-title">{title}: Columns</h4>
</div>
<div class="modal-body">
<p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p>
<p>
<button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button>
<button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button>
</p>
<table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table" data-title="{title}">
<thead>
<tr>
<th class="sorthandle" style="text-align:center;">Sort</th>
<th style="text-align:center;">Visible</th>
<th>Group</th>
<th>Column</th>
<th>Description</th>
<th>ID</th>
<th>Scale</th>
</tr>
</thead>
<tbody>
{trows}
</tbody>
</table>
</div>
<div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div>
</div> </div> </div>""".format(tid=table_id,
title=table_title,
trows="".join(t_modal_headers.values()))
# Save the raw values to a file if requested
if dt.pconfig.get("save_file") is True:
fn = dt.pconfig.get("raw_data_fn", "multiqc_{}".format(table_id))
util_functions.write_data_file(dt.raw_vals, fn)
report.saved_raw_data[fn] = dt.raw_vals
return html
def matplotlib_bargraph (plotdata, plotsamples, pconfig=None):
"""
Plot a bargraph with Matplot lib and return a HTML string. Either embeds a base64
encoded image within HTML or writes the plot and links to it. Should be called by
plot_bargraph, which properly formats the input data.
"""
if pconfig is None:
pconfig = {}
# Plot group ID
if pconfig.get('id') is None:
pconfig['id'] = 'mqc_mplplot_'+''.join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig['id'] = report.save_htmlid(pconfig['id'])
# Individual plot IDs
pids = []
for k in range(len(plotdata)):
try:
name = pconfig['data_labels'][k]
except:
name = k+1
pid = 'mqc_{}_{}'.format(pconfig['id'], name)
pid = report.save_htmlid(pid, skiplint=True)
pids.append(pid)
html = '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> ' + \
'Flat image plot. Toolbox functions such as highlighting / hiding samples will not work ' + \
'(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>'
html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig['id'])
# Same defaults as HighCharts for consistency
default_colors = ['#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9',
'#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1']
# Counts / Percentages Switch
if pconfig.get('cpswitch') is not False and not config.simple_output:
if pconfig.get('cpswitch_c_active', True) is True:
c_active = 'active'
p_active = ''
else:
c_active = ''
p_active = 'active'
pconfig['stacking'] = 'percent'
c_label = pconfig.get('cpswitch_counts_label', 'Counts')
p_label = pconfig.get('cpswitch_percent_label', 'Percentages')
html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_setcountspcnt"> \n\
<button class="btn btn-default btn-sm {c_a} counts">{c_l}</button> \n\
<button class="btn btn-default btn-sm {p_a} pcnt">{p_l}</button> \n\
</div> '.format(c_a=c_active, p_a=p_active, c_l=c_label, p_l=p_label)
if len(plotdata) > 1:
html += ' '
# Buttons to cycle through different datasets
if len(plotdata) > 1 and not config.simple_output:
html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n'
for k, p in enumerate(plotdata):
pid = pids[k]
active = 'active' if k == 0 else ''
try:
name = pconfig['data_labels'][k]
except:
name = k+1
html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format(a=active, pid=pid, n=name)
html += '</div>\n\n'
# Go through datasets creating plots
for pidx, pdata in enumerate(plotdata):
# Save plot data to file
fdata = {}
for d in pdata:
for didx, dval in enumerate(d['data']):
s_name = plotsamples[pidx][didx]
if s_name not in fdata:
fdata[s_name] = dict()
fdata[s_name][d['name']] = dval
util_functions.write_data_file(fdata, pids[pidx])
# Plot percentage as well as counts
plot_pcts = [False]
if pconfig.get('cpswitch') is not False:
plot_pcts = [False, True]
# Switch out NaN for 0s so that MatPlotLib doesn't ignore stuff
for idx, d in enumerate(pdata):
pdata[idx]['data'] = [x if not math.isnan(x) else 0 for x in d['data'] ]
for plot_pct in plot_pcts:
# Plot ID
pid = pids[pidx]
hide_plot = False
if plot_pct is True:
pid = '{}_pc'.format(pid)
if pconfig.get('cpswitch_c_active', True) is True:
hide_plot = True
else:
if pconfig.get('cpswitch_c_active', True) is not True:
hide_plot = True
# Set up figure
plt_height = len(plotsamples[pidx]) / 2.3
plt_height = max(6, plt_height) # At least 6" tall
plt_height = min(30, plt_height) # Cap at 30" tall
bar_width = 0.8
fig = plt.figure(figsize=(14, plt_height), frameon=False)
axes = fig.add_subplot(111)
y_ind = range(len(plotsamples[pidx]))
# Count totals for each sample
if plot_pct is True:
s_totals = [0 for _ in pdata[0]['data']]
for series_idx, d in enumerate(pdata):
for sample_idx, v in enumerate(d['data']):
s_totals[sample_idx] += v
# Plot bars
dlabels = []
for idx, d in enumerate(pdata):
# Plot percentages
values = d['data']
if len(values) < len(y_ind):
values.extend([0] * (len(y_ind) - len(values)))
if plot_pct is True:
for (key,var) in enumerate(values):
s_total = s_totals[key]
if s_total == 0:
values[key] = 0
else:
values[key] = (float(var+0.0)/float(s_total))*100
# Get offset for stacked bars
if idx == 0:
prevdata = [0] * len(plotsamples[pidx])
else:
for i, p in enumerate(prevdata):
prevdata[i] += pdata[idx-1]['data'][i]
# Default colour index
cidx = idx
while cidx >= len(default_colors):
cidx -= len(default_colors)
# Save the name of this series
dlabels.append(d['name'])
# Add the series of bars to the plot
axes.barh(
y_ind,
values,
bar_width,
left = prevdata,
color = d.get('color', default_colors[cidx]),
align = 'center',
linewidth = pconfig.get('borderWidth', 0)
)
# Tidy up axes
axes.tick_params(labelsize=8, direction='out', left=False, right=False, top=False, bottom=False)
axes.set_xlabel(pconfig.get('ylab', '')) # I know, I should fix the fact that the config is switched
axes.set_ylabel(pconfig.get('xlab', ''))
axes.set_yticks(y_ind) # Specify where to put the labels
axes.set_yticklabels(plotsamples[pidx]) # Set y axis sample name labels
axes.set_ylim((-0.5, len(y_ind)-0.5)) # Reduce padding around plot area
if plot_pct is True:
axes.set_xlim((0, 100))
# Add percent symbols
vals = axes.get_xticks()
axes.set_xticklabels(['{:.0f}%'.format(x) for x in vals])
else:
default_xlimits = axes.get_xlim()
axes.set_xlim((pconfig.get('ymin', default_xlimits[0]),pconfig.get('ymax', default_xlimits[1])))
if 'title' in pconfig:
top_gap = 1 + (0.5 / plt_height)
plt.text(0.5, top_gap, pconfig['title'], horizontalalignment='center', fontsize=16, transform=axes.transAxes)
axes.grid(True, zorder=0, which='both', axis='x', linestyle='-', color='#dedede', linewidth=1)
axes.set_axisbelow(True)
axes.spines['right'].set_visible(False)
axes.spines['top'].set_visible(False)
axes.spines['bottom'].set_visible(False)
axes.spines['left'].set_visible(False)
plt.gca().invert_yaxis() # y axis is reverse sorted otherwise
# Hide some labels if we have a lot of samples
show_nth = max(1, math.ceil(len(pdata[0]['data'])/150))
for idx, label in enumerate(axes.get_yticklabels()):
if idx % show_nth != 0:
label.set_visible(False)
# Legend
bottom_gap = -1 * (1 - ((plt_height - 1.5) / plt_height))
lgd = axes.legend(dlabels, loc='lower center', bbox_to_anchor=(0, bottom_gap, 1, .102), ncol=5, mode='expand', fontsize=8, frameon=False)
# Should this plot be hidden on report load?
hidediv = ''
if pidx > 0 or hide_plot:
hidediv = ' style="display:none;"'
# Save the plot to the data directory if export is requested
if config.export_plots:
for fformat in config.export_plot_formats:
# Make the directory if it doesn't already exist
plot_dir = os.path.join(config.plots_dir, fformat)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
# Save the plot
plot_fn = os.path.join(plot_dir, '{}.{}'.format(pid, fformat))
fig.savefig(plot_fn, format=fformat, bbox_extra_artists=(lgd,), bbox_inches='tight')
# Output the figure to a base64 encoded string
if getattr(get_template_mod(), 'base64_plots', True) is True:
img_buffer = io.BytesIO()
fig.savefig(img_buffer, format='png', bbox_inches='tight')
b64_img = base64.b64encode(img_buffer.getvalue()).decode('utf8')
img_buffer.close()
html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format(pid, hidediv, b64_img)
# Link to the saved image
else:
plot_relpath = os.path.join(config.plots_dir_name, 'png', '{}.png'.format(pid))
html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(pid, hidediv, plot_relpath)
plt.close(fig)
# Close wrapping div
html += '</div>'
report.num_mpl_plots += 1
return html
def matplotlib_linegraph(plotdata, pconfig=None):
"""
Plot a line graph with Matplot lib and return a HTML string. Either embeds a base64
encoded image within HTML or writes the plot and links to it. Should be called by
plot_bargraph, which properly formats the input data.
"""
if pconfig is None:
pconfig = {}
# Plot group ID
if pconfig.get("id") is None:
pconfig["id"] = "mqc_mplplot_" + "".join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig["id"] = report.save_htmlid(pconfig["id"])
# Individual plot IDs
pids = []
for k in range(len(plotdata)):
try:
name = pconfig["data_labels"][k]["name"]
except:
name = k + 1
pid = "mqc_{}_{}".format(pconfig["id"], name)
pid = report.save_htmlid(pid, skiplint=True)
pids.append(pid)
html = (
'<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> '
+
"Flat image plot. Toolbox functions such as highlighting / hiding samples will not work "
+
'(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>'
)
html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig["id"])
# Same defaults as HighCharts for consistency
default_colors = [
"#7cb5ec",
"#434348",
"#90ed7d",
"#f7a35c",
"#8085e9",
"#f15c80",
"#e4d354",
"#2b908f",
"#f45b5b",
"#91e8e1",
]
# Buttons to cycle through different datasets
if len(plotdata) > 1 and not config.simple_output:
html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n'
for k, p in enumerate(plotdata):
pid = pids[k]
active = "active" if k == 0 else ""
try:
name = pconfig["data_labels"][k]["name"]
except:
name = k + 1
html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format(
a=active, pid=pid, n=name)
html += "</div>\n\n"
# Go through datasets creating plots
for pidx, pdata in enumerate(plotdata):
# Plot ID
pid = pids[pidx]
# Save plot data to file
fdata = OrderedDict()
lastcats = None
sharedcats = True
for d in pdata:
fdata[d["name"]] = OrderedDict()
for i, x in enumerate(d["data"]):
if type(x) is list:
fdata[d["name"]][str(x[0])] = x[1]
# Check to see if all categories are the same
if lastcats is None:
lastcats = [x[0] for x in d["data"]]
elif lastcats != [x[0] for x in d["data"]]:
sharedcats = False
else:
try:
fdata[d["name"]][pconfig["categories"][i]] = x
except (KeyError, IndexError):
fdata[d["name"]][str(i)] = x
# Custom tsv output if the x axis varies
if not sharedcats and config.data_format == "tsv":
fout = ""
for d in pdata:
fout += "\t" + "\t".join([str(x[0]) for x in d["data"]])
fout += "\n{}\t".format(d["name"])
fout += "\t".join([str(x[1]) for x in d["data"]])
fout += "\n"
with io.open(os.path.join(config.data_dir, "{}.txt".format(pid)),
"w",
encoding="utf-8") as f:
print(fout.encode("utf-8", "ignore").decode("utf-8"), file=f)
else:
util_functions.write_data_file(fdata, pid)
# Set up figure
fig = plt.figure(figsize=(14, 6), frameon=False)
axes = fig.add_subplot(111)
# Go through data series
for idx, d in enumerate(pdata):
# Default colour index
cidx = idx
while cidx >= len(default_colors):
cidx -= len(default_colors)
# Line style
linestyle = "solid"
if d.get("dashStyle", None) == "Dash":
linestyle = "dashed"
# Reformat data (again)
try:
axes.plot(
[x[0] for x in d["data"]],
[x[1] for x in d["data"]],
label=d["name"],
color=d.get("color", default_colors[cidx]),
linestyle=linestyle,
linewidth=1,
marker=None,
)
except TypeError:
# Categorical data on x axis
axes.plot(d["data"],
label=d["name"],
color=d.get("color", default_colors[cidx]),
linewidth=1,
marker=None)
# Tidy up axes
axes.tick_params(labelsize=8,
direction="out",
left=False,
right=False,
top=False,
bottom=False)
axes.set_xlabel(pconfig.get("xlab", ""))
axes.set_ylabel(pconfig.get("ylab", ""))
# Dataset specific y label
try:
axes.set_ylabel(pconfig["data_labels"][pidx]["ylab"])
except:
pass
# Axis limits
default_ylimits = axes.get_ylim()
ymin = default_ylimits[0]
if "ymin" in pconfig:
ymin = pconfig["ymin"]
elif "yFloor" in pconfig:
ymin = max(pconfig["yFloor"], default_ylimits[0])
ymax = default_ylimits[1]
if "ymax" in pconfig:
ymax = pconfig["ymax"]
elif "yCeiling" in pconfig:
ymax = min(pconfig["yCeiling"], default_ylimits[1])
if (ymax - ymin) < pconfig.get("yMinRange", 0):
ymax = ymin + pconfig["yMinRange"]
axes.set_ylim((ymin, ymax))
# Dataset specific ymax
try:
axes.set_ylim((ymin, pconfig["data_labels"][pidx]["ymax"]))
except:
pass
default_xlimits = axes.get_xlim()
xmin = default_xlimits[0]
if "xmin" in pconfig:
xmin = pconfig["xmin"]
elif "xFloor" in pconfig:
xmin = max(pconfig["xFloor"], default_xlimits[0])
xmax = default_xlimits[1]
if "xmax" in pconfig:
xmax = pconfig["xmax"]
elif "xCeiling" in pconfig:
xmax = min(pconfig["xCeiling"], default_xlimits[1])
if (xmax - xmin) < pconfig.get("xMinRange", 0):
xmax = xmin + pconfig["xMinRange"]
axes.set_xlim((xmin, xmax))
# Plot title
if "title" in pconfig:
plt.text(0.5,
1.05,
pconfig["title"],
horizontalalignment="center",
fontsize=16,
transform=axes.transAxes)
axes.grid(True,
zorder=10,
which="both",
axis="y",
linestyle="-",
color="#dedede",
linewidth=1)
# X axis categories, if specified
if "categories" in pconfig:
axes.set_xticks([i for i, v in enumerate(pconfig["categories"])])
axes.set_xticklabels(pconfig["categories"])
# Axis lines
xlim = axes.get_xlim()
axes.plot([xlim[0], xlim[1]], [0, 0],
linestyle="-",
color="#dedede",
linewidth=2)
axes.set_axisbelow(True)
axes.spines["right"].set_visible(False)
axes.spines["top"].set_visible(False)
axes.spines["bottom"].set_visible(False)
axes.spines["left"].set_visible(False)
# Background colours, if specified
if "yPlotBands" in pconfig:
xlim = axes.get_xlim()
for pb in pconfig["yPlotBands"]:
axes.barh(
pb["from"],
xlim[1],
height=pb["to"] - pb["from"],
left=xlim[0],
color=pb["color"],
linewidth=0,
zorder=0,
align="edge",
)
if "xPlotBands" in pconfig:
ylim = axes.get_ylim()
for pb in pconfig["xPlotBands"]:
axes.bar(
pb["from"],
ylim[1],
width=pb["to"] - pb["from"],
bottom=ylim[0],
color=pb["color"],
linewidth=0,
zorder=0,
align="edge",
)
# Tight layout - makes sure that legend fits in and stuff
if len(pdata) <= 15:
axes.legend(
loc="lower center",
bbox_to_anchor=(0, -0.22, 1, 0.102),
ncol=5,
mode="expand",
fontsize=8,
frameon=False,
)
plt.tight_layout(rect=[0, 0.08, 1, 0.92])
else:
plt.tight_layout(rect=[0, 0, 1, 0.92])
# Should this plot be hidden on report load?
hidediv = ""
if pidx > 0:
hidediv = ' style="display:none;"'
# Save the plot to the data directory if export is requests
if config.export_plots:
for fformat in config.export_plot_formats:
# Make the directory if it doesn't already exist
plot_dir = os.path.join(config.plots_dir, fformat)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
# Save the plot
plot_fn = os.path.join(plot_dir, "{}.{}".format(pid, fformat))
fig.savefig(plot_fn, format=fformat, bbox_inches="tight")
# Output the figure to a base64 encoded string
if getattr(get_template_mod(), "base64_plots", True) is True:
img_buffer = io.BytesIO()
fig.savefig(img_buffer, format="png", bbox_inches="tight")
b64_img = base64.b64encode(img_buffer.getvalue()).decode("utf8")
img_buffer.close()
html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format(
pid, hidediv, b64_img)
# Save to a file and link <img>
else:
plot_relpath = os.path.join(config.plots_dir_name, "png",
"{}.png".format(pid))
html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(
pid, hidediv, plot_relpath)
plt.close(fig)
# Close wrapping div
html += "</div>"
report.num_mpl_plots += 1
return html
def highcharts_bargraph(plotdata, plotsamples=None, pconfig=None):
"""
Build the HTML needed for a HighCharts bar graph. Should be
called by plot_bargraph, which properly formats input data.
"""
if pconfig is None:
pconfig = {}
if pconfig.get("id") is None:
pconfig["id"] = "mqc_hcplot_" + "".join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig["id"] = report.save_htmlid(pconfig["id"])
html = '<div class="mqc_hcplot_plotgroup">'
# Counts / Percentages / Log Switches
if pconfig.get("cpswitch") is not False or pconfig.get("logswitch") is True:
if pconfig.get("logswitch_active") is True:
c_active = ""
p_active = ""
l_active = "active"
elif pconfig.get("cpswitch_c_active", True) is True:
c_active = "active"
p_active = ""
l_active = ""
else:
c_active = ""
p_active = "active"
l_active = ""
pconfig["stacking"] = "percent"
c_label = pconfig.get("cpswitch_counts_label", "Counts")
p_label = pconfig.get("cpswitch_percent_label", "Percentages")
l_label = pconfig.get("logswitch_label", "Log10")
html += '<div class="btn-group hc_switch_group"> \n'
html += '<button class="btn btn-default btn-sm {c_a}" data-action="set_numbers" data-target="{id}" data-ylab="{c_l}">{c_l}</button> \n'.format(
id=pconfig["id"], c_a=c_active, c_l=c_label
)
if pconfig.get("cpswitch", True) is True:
html += '<button class="btn btn-default btn-sm {p_a}" data-action="set_percent" data-target="{id}" data-ylab="{p_l}">{p_l}</button> \n'.format(
id=pconfig["id"], p_a=p_active, p_l=p_label
)
if pconfig.get("logswitch") is True:
html += '<button class="btn btn-default btn-sm {l_a}" data-action="set_log" data-target="{id}" data-ylab="{l_l}">{l_l}</button> \n'.format(
id=pconfig["id"], l_a=l_active, l_l=l_label
)
pconfig["reversedStacks"] = True
html += "</div> "
if len(plotdata) > 1:
html += " "
# Buttons to cycle through different datasets
if len(plotdata) > 1:
html += '<div class="btn-group hc_switch_group">\n'
for k, p in enumerate(plotdata):
active = "active" if k == 0 else ""
try:
name = pconfig["data_labels"][k]["name"]
except:
try:
name = pconfig["data_labels"][k]
except:
name = k + 1
try:
ylab = 'data-ylab="{}"'.format(pconfig["data_labels"][k]["ylab"])
except:
ylab = 'data-ylab="{}"'.format(name) if name != k + 1 else ""
try:
ymax = 'data-ymax="{}"'.format(pconfig["data_labels"][k]["ymax"])
except:
ymax = ""
html += '<button class="btn btn-default btn-sm {a}" data-action="set_data" {y} {ym} data-newdata="{k}" data-target="{id}">{n}</button>\n'.format(
a=active, id=pconfig["id"], n=name, y=ylab, ym=ymax, k=k
)
html += "</div>\n\n"
# Plot HTML
html += """<div class="hc-plot-wrapper"{height}>
<div id="{id}" class="hc-plot not_rendered hc-bar-plot"><small>loading..</small></div>
</div></div>""".format(
id=pconfig["id"],
height=f' style="height:{pconfig["height"]}px"' if "height" in pconfig else "",
)
report.num_hc_plots += 1
report.plot_data[pconfig["id"]] = {
"plot_type": "bar_graph",
"samples": plotsamples,
"datasets": plotdata,
"config": pconfig,
}
return html
def matplotlib_bargraph(plotdata, plotsamples, pconfig=None):
"""
Plot a bargraph with Matplot lib and return a HTML string. Either embeds a base64
encoded image within HTML or writes the plot and links to it. Should be called by
plot_bargraph, which properly formats the input data.
"""
if pconfig is None:
pconfig = {}
# Plot group ID
if pconfig.get("id") is None:
pconfig["id"] = "mqc_mplplot_" + "".join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig["id"] = report.save_htmlid(pconfig["id"])
# Individual plot IDs
pids = []
for k in range(len(plotdata)):
try:
name = pconfig["data_labels"][k]
except:
name = k + 1
pid = "mqc_{}_{}".format(pconfig["id"], name)
pid = report.save_htmlid(pid, skiplint=True)
pids.append(pid)
html = (
'<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> '
+ "Flat image plot. Toolbox functions such as highlighting / hiding samples will not work "
+ '(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>'
)
html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig["id"])
# Same defaults as HighCharts for consistency
default_colors = [
"#7cb5ec",
"#434348",
"#90ed7d",
"#f7a35c",
"#8085e9",
"#f15c80",
"#e4d354",
"#2b908f",
"#f45b5b",
"#91e8e1",
]
# Counts / Percentages Switch
if pconfig.get("cpswitch") is not False and not config.simple_output:
if pconfig.get("cpswitch_c_active", True) is True:
c_active = "active"
p_active = ""
else:
c_active = ""
p_active = "active"
pconfig["stacking"] = "percent"
c_label = pconfig.get("cpswitch_counts_label", "Counts")
p_label = pconfig.get("cpswitch_percent_label", "Percentages")
html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_setcountspcnt"> \n\
<button class="btn btn-default btn-sm {c_a} counts">{c_l}</button> \n\
<button class="btn btn-default btn-sm {p_a} pcnt">{p_l}</button> \n\
</div> '.format(
c_a=c_active, p_a=p_active, c_l=c_label, p_l=p_label
)
if len(plotdata) > 1:
html += " "
# Buttons to cycle through different datasets
if len(plotdata) > 1 and not config.simple_output:
html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n'
for k, p in enumerate(plotdata):
pid = pids[k]
active = "active" if k == 0 else ""
try:
name = pconfig["data_labels"][k]
except:
name = k + 1
html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format(
a=active, pid=pid, n=name
)
html += "</div>\n\n"
# Go through datasets creating plots
for pidx, pdata in enumerate(plotdata):
# Save plot data to file
fdata = {}
for d in pdata:
for didx, dval in enumerate(d["data"]):
s_name = plotsamples[pidx][didx]
if s_name not in fdata:
fdata[s_name] = dict()
fdata[s_name][d["name"]] = dval
if pconfig.get("save_data_file", True):
util_functions.write_data_file(fdata, pids[pidx])
# Plot percentage as well as counts
plot_pcts = [False]
if pconfig.get("cpswitch") is not False:
plot_pcts = [False, True]
# Switch out NaN for 0s so that MatPlotLib doesn't ignore stuff
for idx, d in enumerate(pdata):
pdata[idx]["data"] = [x if not math.isnan(x) else 0 for x in d["data"]]
for plot_pct in plot_pcts:
# Plot ID
pid = pids[pidx]
hide_plot = False
if plot_pct is True:
pid = "{}_pc".format(pid)
if pconfig.get("cpswitch_c_active", True) is True:
hide_plot = True
else:
if pconfig.get("cpswitch_c_active", True) is not True:
hide_plot = True
# Set up figure
# Height has a default, then adjusted by the number of samples
plt_height = len(plotsamples[pidx]) / 2.3 # Default in inches, empirically determined
plt_height = max(6, plt_height) # At least 6" tall
plt_height = min(30, plt_height) # Cap at 30" tall
# Use fixed height if pconfig['height'] is set (convert pixels -> inches)
if "height" in pconfig:
# Default interactive height in pixels = 512
# Not perfect replication, but good enough
plt_height = 6 * (pconfig["height"] / 512)
bar_width = 0.8
fig = plt.figure(figsize=(14, plt_height), frameon=False)
axes = fig.add_subplot(111)
y_ind = range(len(plotsamples[pidx]))
# Count totals for each sample
if plot_pct is True:
s_totals = [0 for _ in pdata[0]["data"]]
for series_idx, d in enumerate(pdata):
for sample_idx, v in enumerate(d["data"]):
s_totals[sample_idx] += v
# Plot bars
dlabels = []
prev_values = None
for idx, d in enumerate(pdata):
# Plot percentages
values = [x for x in d["data"]]
if len(values) < len(y_ind):
values.extend([0] * (len(y_ind) - len(values)))
if plot_pct is True:
for (key, var) in enumerate(values):
s_total = s_totals[key]
if s_total == 0:
values[key] = 0
else:
values[key] = (float(var + 0.0) / float(s_total)) * 100
# Get offset for stacked bars
if idx == 0:
prevdata = [0] * len(plotsamples[pidx])
else:
for i, p in enumerate(prevdata):
prevdata[i] += prev_values[i]
# Default colour index
cidx = idx
while cidx >= len(default_colors):
cidx -= len(default_colors)
# Save the name of this series
dlabels.append(d["name"])
# Add the series of bars to the plot
axes.barh(
y_ind,
values,
bar_width,
left=prevdata,
color=d.get("color", default_colors[cidx]),
align="center",
linewidth=pconfig.get("borderWidth", 0),
)
prev_values = values
# Tidy up axes
axes.tick_params(
labelsize=pconfig.get("labelSize", 8), direction="out", left=False, right=False, top=False, bottom=False
)
axes.set_xlabel(pconfig.get("ylab", "")) # I know, I should fix the fact that the config is switched
axes.set_ylabel(pconfig.get("xlab", ""))
axes.set_yticks(y_ind) # Specify where to put the labels
axes.set_yticklabels(plotsamples[pidx]) # Set y axis sample name labels
axes.set_ylim((-0.5, len(y_ind) - 0.5)) # Reduce padding around plot area
if plot_pct is True:
axes.set_xlim((0, 100))
# Add percent symbols
vals = axes.get_xticks()
axes.set_xticks(axes.get_xticks())
axes.set_xticklabels(["{:.0f}%".format(x) for x in vals])
else:
default_xlimits = axes.get_xlim()
axes.set_xlim((pconfig.get("ymin", default_xlimits[0]), pconfig.get("ymax", default_xlimits[1])))
if "title" in pconfig:
top_gap = 1 + (0.5 / plt_height)
plt.text(
0.5, top_gap, pconfig["title"], horizontalalignment="center", fontsize=16, transform=axes.transAxes
)
axes.grid(True, zorder=0, which="both", axis="x", linestyle="-", color="#dedede", linewidth=1)
axes.set_axisbelow(True)
axes.spines["right"].set_visible(False)
axes.spines["top"].set_visible(False)
axes.spines["bottom"].set_visible(False)
axes.spines["left"].set_visible(False)
plt.gca().invert_yaxis() # y axis is reverse sorted otherwise
# Hide some labels if we have a lot of samples
show_nth = max(1, math.ceil(len(pdata[0]["data"]) / 150))
for idx, label in enumerate(axes.get_yticklabels()):
if idx % show_nth != 0:
label.set_visible(False)
# Legend
bottom_gap = -1 * (1 - ((plt_height - 1.5) / plt_height))
lgd = axes.legend(
dlabels,
loc="lower center",
bbox_to_anchor=(0, bottom_gap, 1, 0.102),
ncol=5,
mode="expand",
fontsize=pconfig.get("labelSize", 8),
frameon=False,
)
# Should this plot be hidden on report load?
hidediv = ""
if pidx > 0 or hide_plot:
hidediv = ' style="display:none;"'
# Save the plot to the data directory if export is requested
if config.export_plots:
for fformat in config.export_plot_formats:
# Make the directory if it doesn't already exist
plot_dir = os.path.join(config.plots_dir, fformat)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
# Save the plot
plot_fn = os.path.join(plot_dir, "{}.{}".format(pid, fformat))
fig.savefig(plot_fn, format=fformat, bbox_extra_artists=(lgd,), bbox_inches="tight")
# Output the figure to a base64 encoded string
if getattr(get_template_mod(), "base64_plots", True) is True:
img_buffer = io.BytesIO()
fig.savefig(img_buffer, format="png", bbox_inches="tight")
b64_img = base64.b64encode(img_buffer.getvalue()).decode("utf8")
img_buffer.close()
html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format(
pid, hidediv, b64_img
)
# Link to the saved image
else:
plot_relpath = os.path.join(config.plots_dir_name, "png", "{}.png".format(pid))
html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(pid, hidediv, plot_relpath)
plt.close(fig)
# Close wrapping div
html += "</div>"
return html
def sequence_content_plot(self):
"""Create the epic HTML for the FastQC sequence content heatmap"""
# Prep the data
data = OrderedDict()
for s_name in sorted(self.fastqc_data.keys()):
try:
data[s_name] = {
self.avg_bp_from_range(d["base"]): d for d in self.fastqc_data[s_name]["per_base_sequence_content"]
}
except KeyError:
# FastQC module was skipped - move on to the next sample
continue
# Old versions of FastQC give counts instead of percentages
for b in data[s_name]:
tot = sum([data[s_name][b][base] for base in ["a", "c", "t", "g"]])
if tot == 100.0:
break # Stop loop after one iteration if summed to 100 (percentages)
else:
for base in ["a", "c", "t", "g"]:
data[s_name][b][base] = (float(data[s_name][b][base]) / float(tot)) * 100.0
# Replace NaN with 0
for b in data[s_name]:
for base in ["a", "c", "t", "g"]:
if math.isnan(float(data[s_name][b][base])):
data[s_name][b][base] = 0
if len(data) == 0:
log.debug("sequence_content not found in FastQC reports")
return None
html = """<div id="fastqc_per_base_sequence_content_plot_div">
<div class="alert alert-info">
<span class="glyphicon glyphicon-hand-up"></span>
Click a sample row to see a line plot for that dataset.
</div>
<h5><span class="s_name text-primary"><span class="glyphicon glyphicon-info-sign"></span> Rollover for sample name</span></h5>
<button id="fastqc_per_base_sequence_content_export_btn"><span class="glyphicon glyphicon-download-alt"></span> Export Plot</button>
<div class="fastqc_seq_heatmap_key">
Position: <span id="fastqc_seq_heatmap_key_pos">-</span>
<div><span id="fastqc_seq_heatmap_key_t"> %T: <span>-</span></span></div>
<div><span id="fastqc_seq_heatmap_key_c"> %C: <span>-</span></span></div>
<div><span id="fastqc_seq_heatmap_key_a"> %A: <span>-</span></span></div>
<div><span id="fastqc_seq_heatmap_key_g"> %G: <span>-</span></span></div>
</div>
<div id="fastqc_seq_heatmap_div" class="fastqc-overlay-plot">
<div id="{id}" class="fastqc_per_base_sequence_content_plot hc-plot has-custom-export">
<canvas id="fastqc_seq_heatmap" height="100%" width="800px" style="width:100%;"></canvas>
</div>
</div>
<div class="clearfix"></div>
</div>
<script type="application/json" class="fastqc_seq_content">{d}</script>
""".format(
# Generate unique plot ID, needed in mqc_export_selectplots
id=report.save_htmlid("fastqc_per_base_sequence_content_plot"),
d=json.dumps([self.anchor.replace("-", "_"), data]),
)
self.add_section(
name="Per Base Sequence Content",
anchor="fastqc_per_base_sequence_content",
description="The proportion of each base position for which each of the four normal DNA bases has been called.",
helptext="""
To enable multiple samples to be shown in a single plot, the base composition data
is shown as a heatmap. The colours represent the balance between the four bases:
an even distribution should give an even muddy brown colour. Hover over the plot
to see the percentage of the four bases under the cursor.
**To see the data as a line plot, as in the original FastQC graph, click on a sample track.**
From the [FastQC help](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/3%20Analysis%20Modules/4%20Per%20Base%20Sequence%20Content.html):
_Per Base Sequence Content plots out the proportion of each base position in a
file for which each of the four normal DNA bases has been called._
_In a random library you would expect that there would be little to no difference
between the different bases of a sequence run, so the lines in this plot should
run parallel with each other. The relative amount of each base should reflect
the overall amount of these bases in your genome, but in any case they should
not be hugely imbalanced from each other._
_It's worth noting that some types of library will always produce biased sequence
composition, normally at the start of the read. Libraries produced by priming
using random hexamers (including nearly all RNA-Seq libraries) and those which
were fragmented using transposases inherit an intrinsic bias in the positions
at which reads start. This bias does not concern an absolute sequence, but instead
provides enrichement of a number of different K-mers at the 5' end of the reads.
Whilst this is a true technical bias, it isn't something which can be corrected
by trimming and in most cases doesn't seem to adversely affect the downstream
analysis._
""",
content=html,
)
def highcharts_scatter_plot(plotdata, pconfig=None):
"""
Build the HTML needed for a HighCharts scatter plot. Should be
called by scatter.plot(), which properly formats input data.
"""
if pconfig is None:
pconfig = {}
# Get the plot ID
if pconfig.get("id") is None:
pconfig["id"] = "mqc_hcplot_" + "".join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig["id"] = report.save_htmlid(pconfig["id"])
# Build the HTML for the page
html = '<div class="mqc_hcplot_plotgroup">'
# Buttons to cycle through different datasets
if len(plotdata) > 1:
html += '<div class="btn-group hc_switch_group">\n'
for k, p in enumerate(plotdata):
active = "active" if k == 0 else ""
try:
name = pconfig["data_labels"][k]["name"]
except:
name = k + 1
try:
ylab = 'data-ylab="{}"'.format(
pconfig["data_labels"][k]["ylab"])
except:
ylab = 'data-ylab="{}"'.format(name) if name != k + 1 else ""
try:
ymax = 'data-ymax="{}"'.format(
pconfig["data_labels"][k]["ymax"])
except:
ymax = ""
try:
xlab = 'data-xlab="{}"'.format(
pconfig["data_labels"][k]["xlab"])
except:
xlab = 'data-xlab="{}"'.format(name) if name != k + 1 else ""
html += '<button class="btn btn-default btn-sm {a}" data-action="set_data" {y} {ym} {xl} data-newdata="{k}" data-target="{id}">{n}</button>\n'.format(
a=active,
id=pconfig["id"],
n=name,
y=ylab,
ym=ymax,
xl=xlab,
k=k)
html += "</div>\n\n"
# The plot div
html += '<div class="hc-plot-wrapper"{height}><div id="{id}" class="hc-plot not_rendered hc-scatter-plot"><small>loading..</small></div></div></div> \n'.format(
id=pconfig["id"],
height=f' style="height:{pconfig["height"]}px"'
if "height" in pconfig else "",
)
report.num_hc_plots += 1
report.plot_data[pconfig["id"]] = {
"plot_type": "scatter",
"datasets": plotdata,
"config": pconfig
}
return html
def matplotlib_bargraph(plotdata, plotsamples, pconfig=None):
"""
Plot a bargraph with Matplot lib and return a HTML string. Either embeds a base64
encoded image within HTML or writes the plot and links to it. Should be called by
plot_bargraph, which properly formats the input data.
"""
if pconfig is None:
pconfig = {}
# Plot group ID
if pconfig.get('id') is None:
pconfig['id'] = 'mqc_mplplot_' + ''.join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig['id'] = report.save_htmlid(pconfig['id'])
# Individual plot IDs
pids = []
for k in range(len(plotdata)):
try:
name = pconfig['data_labels'][k]
except:
name = k + 1
pid = 'mqc_{}_{}'.format(pconfig['id'], name)
pid = report.save_htmlid(pid, skiplint=True)
pids.append(pid)
html = '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> ' + \
'Flat image plot. Toolbox functions such as highlighting / hiding samples will not work ' + \
'(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>'
html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig['id'])
# Same defaults as HighCharts for consistency
default_colors = [
'#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9', '#f15c80',
'#e4d354', '#2b908f', '#f45b5b', '#91e8e1'
]
# Counts / Percentages Switch
if pconfig.get('cpswitch') is not False and not config.simple_output:
if pconfig.get('cpswitch_c_active', True) is True:
c_active = 'active'
p_active = ''
else:
c_active = ''
p_active = 'active'
pconfig['stacking'] = 'percent'
c_label = pconfig.get('cpswitch_counts_label', 'Counts')
p_label = pconfig.get('cpswitch_percent_label', 'Percentages')
html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_setcountspcnt"> \n\
<button class="btn btn-default btn-sm {c_a} counts">{c_l}</button> \n\
<button class="btn btn-default btn-sm {p_a} pcnt">{p_l}</button> \n\
</div> '.format(c_a=c_active, p_a=p_active, c_l=c_label, p_l=p_label)
if len(plotdata) > 1:
html += ' '
# Buttons to cycle through different datasets
if len(plotdata) > 1 and not config.simple_output:
html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n'
for k, p in enumerate(plotdata):
pid = pids[k]
active = 'active' if k == 0 else ''
try:
name = pconfig['data_labels'][k]
except:
name = k + 1
html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format(
a=active, pid=pid, n=name)
html += '</div>\n\n'
# Go through datasets creating plots
for pidx, pdata in enumerate(plotdata):
# Save plot data to file
fdata = {}
for d in pdata:
for didx, dval in enumerate(d['data']):
s_name = plotsamples[pidx][didx]
if s_name not in fdata:
fdata[s_name] = dict()
fdata[s_name][d['name']] = dval
util_functions.write_data_file(fdata, pids[pidx])
# Plot percentage as well as counts
plot_pcts = [False]
if pconfig.get('cpswitch') is not False:
plot_pcts = [False, True]
# Switch out NaN for 0s so that MatPlotLib doesn't ignore stuff
for idx, d in enumerate(pdata):
pdata[idx]['data'] = [
x if not math.isnan(x) else 0 for x in d['data']
]
for plot_pct in plot_pcts:
# Plot ID
pid = pids[pidx]
hide_plot = False
if plot_pct is True:
pid = '{}_pc'.format(pid)
if pconfig.get('cpswitch_c_active', True) is True:
hide_plot = True
else:
if pconfig.get('cpswitch_c_active', True) is not True:
hide_plot = True
# Set up figure
plt_height = len(plotsamples[pidx]) / 2.3
plt_height = max(6, plt_height) # At least 6" tall
plt_height = min(30, plt_height) # Cap at 30" tall
bar_width = 0.8
fig = plt.figure(figsize=(14, plt_height), frameon=False)
axes = fig.add_subplot(111)
y_ind = range(len(plotsamples[pidx]))
# Count totals for each sample
if plot_pct is True:
s_totals = [0 for _ in pdata[0]['data']]
for series_idx, d in enumerate(pdata):
for sample_idx, v in enumerate(d['data']):
s_totals[sample_idx] += v
# Plot bars
dlabels = []
for idx, d in enumerate(pdata):
# Plot percentages
values = d['data']
if len(values) < len(y_ind):
values.extend([0] * (len(y_ind) - len(values)))
if plot_pct is True:
for (key, var) in enumerate(values):
s_total = s_totals[key]
if s_total == 0:
values[key] = 0
else:
values[key] = (float(var + 0.0) /
float(s_total)) * 100
# Get offset for stacked bars
if idx == 0:
prevdata = [0] * len(plotsamples[pidx])
else:
for i, p in enumerate(prevdata):
prevdata[i] += pdata[idx - 1]['data'][i]
# Default colour index
cidx = idx
while cidx >= len(default_colors):
cidx -= len(default_colors)
# Save the name of this series
dlabels.append(d['name'])
# Add the series of bars to the plot
axes.barh(y_ind,
values,
bar_width,
left=prevdata,
color=d.get('color', default_colors[cidx]),
align='center',
linewidth=pconfig.get('borderWidth', 0))
# Tidy up axes
axes.tick_params(labelsize=8,
direction='out',
left=False,
right=False,
top=False,
bottom=False)
axes.set_xlabel(
pconfig.get('ylab', '')
) # I know, I should fix the fact that the config is switched
axes.set_ylabel(pconfig.get('xlab', ''))
axes.set_yticks(y_ind) # Specify where to put the labels
axes.set_yticklabels(
plotsamples[pidx]) # Set y axis sample name labels
axes.set_ylim(
(-0.5, len(y_ind) - 0.5)) # Reduce padding around plot area
if plot_pct is True:
axes.set_xlim((0, 100))
# Add percent symbols
vals = axes.get_xticks()
axes.set_xticklabels(['{:.0f}%'.format(x) for x in vals])
else:
default_xlimits = axes.get_xlim()
axes.set_xlim((pconfig.get('ymin', default_xlimits[0]),
pconfig.get('ymax', default_xlimits[1])))
if 'title' in pconfig:
top_gap = 1 + (0.5 / plt_height)
plt.text(0.5,
top_gap,
pconfig['title'],
horizontalalignment='center',
fontsize=16,
transform=axes.transAxes)
axes.grid(True,
zorder=0,
which='both',
axis='x',
linestyle='-',
color='#dedede',
linewidth=1)
axes.set_axisbelow(True)
axes.spines['right'].set_visible(False)
axes.spines['top'].set_visible(False)
axes.spines['bottom'].set_visible(False)
axes.spines['left'].set_visible(False)
plt.gca().invert_yaxis() # y axis is reverse sorted otherwise
# Hide some labels if we have a lot of samples
show_nth = max(1, math.ceil(len(pdata[0]['data']) / 150))
for idx, label in enumerate(axes.get_yticklabels()):
if idx % show_nth != 0:
label.set_visible(False)
# Legend
bottom_gap = -1 * (1 - ((plt_height - 1.5) / plt_height))
lgd = axes.legend(dlabels,
loc='lower center',
bbox_to_anchor=(0, bottom_gap, 1, .102),
ncol=5,
mode='expand',
fontsize=8,
frameon=False)
# Should this plot be hidden on report load?
hidediv = ''
if pidx > 0 or hide_plot:
hidediv = ' style="display:none;"'
# Save the plot to the data directory if export is requested
if config.export_plots:
for fformat in config.export_plot_formats:
# Make the directory if it doesn't already exist
plot_dir = os.path.join(config.plots_dir, fformat)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
# Save the plot
plot_fn = os.path.join(plot_dir,
'{}.{}'.format(pid, fformat))
fig.savefig(plot_fn,
format=fformat,
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
# Output the figure to a base64 encoded string
if getattr(get_template_mod(), 'base64_plots', True) is True:
img_buffer = io.BytesIO()
fig.savefig(img_buffer, format='png', bbox_inches='tight')
b64_img = base64.b64encode(
img_buffer.getvalue()).decode('utf8')
img_buffer.close()
html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format(
pid, hidediv, b64_img)
# Link to the saved image
else:
plot_relpath = os.path.join(config.plots_dir_name, 'png',
'{}.png'.format(pid))
html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(
pid, hidediv, plot_relpath)
plt.close(fig)
# Close wrapping div
html += '</div>'
report.num_mpl_plots += 1
return html
def highcharts_bargraph(plotdata, plotsamples=None, pconfig=None):
"""
Build the HTML needed for a HighCharts bar graph. Should be
called by plot_bargraph, which properly formats input data.
"""
if pconfig is None:
pconfig = {}
if pconfig.get('id') is None:
pconfig['id'] = 'mqc_hcplot_' + ''.join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig['id'] = report.save_htmlid(pconfig['id'])
html = '<div class="mqc_hcplot_plotgroup">'
# Counts / Percentages / Log Switches
if pconfig.get(
'cpswitch') is not False or pconfig.get('logswitch') is True:
if pconfig.get('cpswitch_c_active', True) is True:
c_active = 'active'
p_active = ''
l_active = ''
elif pconfig.get('logswitch_active') is True:
c_active = ''
p_active = ''
l_active = 'active'
else:
c_active = ''
p_active = 'active'
l_active = ''
pconfig['stacking'] = 'percent'
c_label = pconfig.get('cpswitch_counts_label', 'Counts')
p_label = pconfig.get('cpswitch_percent_label', 'Percentages')
l_label = pconfig.get('logswitch_label', 'Log10')
html += '<div class="btn-group hc_switch_group"> \n'
html += '<button class="btn btn-default btn-sm {c_a}" data-action="set_numbers" data-target="{id}" data-ylab="{c_l}">{c_l}</button> \n'.format(
id=pconfig['id'], c_a=c_active, c_l=c_label)
if pconfig.get('cpswitch', True) is True:
html += '<button class="btn btn-default btn-sm {p_a}" data-action="set_percent" data-target="{id}" data-ylab="{p_l}">{p_l}</button> \n'.format(
id=pconfig['id'], p_a=p_active, p_l=p_label)
if pconfig.get('logswitch') is True:
html += '<button class="btn btn-default btn-sm {l_a}" data-action="set_log" data-target="{id}" data-ylab="{l_l}">{l_l}</button> \n'.format(
id=pconfig['id'], l_a=l_active, l_l=l_label)
pconfig['reversedStacks'] = True
html += '</div> '
if len(plotdata) > 1:
html += ' '
# Buttons to cycle through different datasets
if len(plotdata) > 1:
html += '<div class="btn-group hc_switch_group">\n'
for k, p in enumerate(plotdata):
active = 'active' if k == 0 else ''
try:
name = pconfig['data_labels'][k]['name']
except:
try:
name = pconfig['data_labels'][k]
except:
name = k + 1
try:
ylab = 'data-ylab="{}"'.format(
pconfig['data_labels'][k]['ylab'])
except:
ylab = 'data-ylab="{}"'.format(name) if name != k + 1 else ''
try:
ymax = 'data-ymax="{}"'.format(
pconfig['data_labels'][k]['ymax'])
except:
ymax = ''
html += '<button class="btn btn-default btn-sm {a}" data-action="set_data" {y} {ym} data-newdata="{k}" data-target="{id}">{n}</button>\n'.format(
a=active, id=pconfig['id'], n=name, y=ylab, ym=ymax, k=k)
html += '</div>\n\n'
# Plot HTML
html += """<div class="hc-plot-wrapper">
<div id="{id}" class="hc-plot not_rendered hc-bar-plot"><small>loading..</small></div>
</div></div>""".format(id=pconfig['id'])
report.num_hc_plots += 1
report.plot_data[pconfig['id']] = {
'plot_type': 'bar_graph',
'samples': plotsamples,
'datasets': plotdata,
'config': pconfig
}
return html
def fqscreen_plot(self):
"""Makes a fancy custom plot which replicates the plot seen in the main
FastQ Screen program. Not useful if lots of samples as gets too wide."""
categories = list()
getCats = True
data = list()
p_types = OrderedDict()
p_types["multiple_hits_multiple_libraries"] = {
"col": "#7f0000",
"name": "Multiple Hits, Multiple Genomes"
}
p_types["one_hit_multiple_libraries"] = {
"col": "#ff0000",
"name": "One Hit, Multiple Genomes"
}
p_types["multiple_hits_one_library"] = {
"col": "#00007f",
"name": "Multiple Hits, One Genome"
}
p_types["one_hit_one_library"] = {
"col": "#0000ff",
"name": "One Hit, One Genome"
}
for k, t in p_types.items():
first = True
for s in sorted(self.fq_screen_data.keys()):
thisdata = list()
if len(categories) > 0:
getCats = False
for org in sorted(self.fq_screen_data[s]):
if org == "total_reads":
continue
try:
thisdata.append(
self.fq_screen_data[s][org]["percentages"][k])
except KeyError:
thisdata.append(None)
if getCats:
categories.append(org)
td = {
"name": t["name"],
"stack": s,
"data": thisdata,
"color": t["col"]
}
if first:
first = False
else:
td["linkedTo"] = ":previous"
data.append(td)
plot_id = report.save_htmlid("fq_screen_plot")
html = """<div id={plot_id} class="fq_screen_plot hc-plot"></div>
<script type="application/json" class="fq_screen_dict">{dict}</script>
""".format(
plot_id=json.dumps(plot_id),
dict=json.dumps({
"plot_id": plot_id,
"data": data,
"categories": categories
}),
)
html += """<script type="text/javascript">
fq_screen_dict = { }; // { <plot_id>: data, categories }
$('.fq_screen_dict').each(function (i, elem) {
var dict = JSON.parse(elem.innerHTML);
fq_screen_dict[dict.plot_id] = dict;
});
$(function () {
// In case of repeated modules: #fq_screen_plot, #fq_screen_plot-1, ..
$(".fq_screen_plot").each(function () {
var plot_id = $(this).attr('id');
$(this).highcharts({
chart: { type: "column", backgroundColor: null },
title: { text: "FastQ Screen Results" },
xAxis: { categories: fq_screen_dict[plot_id].categories },
yAxis: {
max: 100,
min: 0,
title: { text: "Percentage Aligned" }
},
tooltip: {
formatter: function () {
return "<b>" + this.series.stackKey.replace("column","") + " - " + this.x + "</b><br/>" +
this.series.name + ": " + this.y + "%<br/>" +
"Total Alignment: " + this.point.stackTotal + "%";
},
},
plotOptions: {
column: {
pointPadding: 0,
groupPadding: 0.02,
stacking: "normal"
}
},
series: fq_screen_dict[plot_id].data
});
});
});
</script>"""
return html
def __init__ (self, data, headers=None, pconfig=None):
""" Prepare data for use in a table or plot """
if headers is None:
headers = []
if pconfig is None:
pconfig = {}
# Given one dataset - turn it into a list
if type(data) is not list:
data = [data]
if type(headers) is not list:
headers = [headers]
sectcols = ['55,126,184', '77,175,74', '152,78,163', '255,127,0', '228,26,28', '255,255,51', '166,86,40', '247,129,191', '153,153,153']
shared_keys = defaultdict(lambda: dict())
# Go through each table section
for idx, d in enumerate(data):
# Get the header keys
try:
keys = headers[idx].keys()
assert len(keys) > 0
except (IndexError, AttributeError, AssertionError):
keys = list()
for samp in d.values():
for k in samp.keys():
if k not in keys:
keys.append(k)
try:
headers[idx]
except IndexError:
headers.append(list)
headers[idx] = OrderedDict()
for k in keys:
headers[idx][k] = {}
# Ensure that keys are strings, not numeric
keys = [str(k) for k in keys]
for k in list(headers[idx].keys()):
headers[idx][str(k)] = headers[idx].pop(k)
# Ensure that all sample names are strings as well
cdata = OrderedDict()
for k,v in data[idx].items():
cdata[str(k)] = v
data[idx] = cdata
for s_name in data[idx].keys():
for k in list(data[idx][s_name].keys()):
data[idx][s_name][str(k)] = data[idx][s_name].pop(k)
# Check that we have some data in each column
empties = list()
for k in keys:
n = 0
for samp in d.values():
if k in samp:
n += 1
if n == 0:
empties.append(k)
for k in empties:
keys = [j for j in keys if j != k]
del headers[idx][k]
for k in keys:
# Unique id to avoid overwriting by other datasets
if 'rid' not in headers[idx][k]:
headers[idx][k]['rid'] = report.save_htmlid(re.sub(r'\W+', '_', k).strip().strip('_'))
# Applying defaults presets for data keys if shared_key is set to base_count or read_count
shared_key = headers[idx][k].get('shared_key', None)
if shared_key in ['read_count', 'base_count']:
if shared_key == 'read_count':
multiplier = config.read_count_multiplier
else:
multiplier = config.base_count_multiplier
if headers[idx][k].get('modify') is None:
headers[idx][k]['modify'] = lambda x: x * multiplier
if headers[idx][k].get('min') is None:
headers[idx][k]['min'] = 0
if headers[idx][k].get('format') is None:
if multiplier == 1:
headers[idx][k]['format'] = '{:,.0f}'
# Use defaults / data keys if headers not given
headers[idx][k]['namespace'] = headers[idx][k].get('namespace', pconfig.get('namespace', ''))
headers[idx][k]['title'] = headers[idx][k].get('title', k)
headers[idx][k]['description'] = headers[idx][k].get('description', headers[idx][k]['title'])
headers[idx][k]['scale'] = headers[idx][k].get('scale', pconfig.get('scale', 'GnBu'))
headers[idx][k]['format'] = headers[idx][k].get('format', pconfig.get('format', '{:,.1f}'))
headers[idx][k]['colour'] = headers[idx][k].get('colour', pconfig.get('colour', None))
headers[idx][k]['hidden'] = headers[idx][k].get('hidden', pconfig.get('hidden', None))
headers[idx][k]['max'] = headers[idx][k].get('max', pconfig.get('max', None))
headers[idx][k]['min'] = headers[idx][k].get('min', pconfig.get('min', None))
headers[idx][k]['ceiling'] = headers[idx][k].get('ceiling', pconfig.get('ceiling', None))
headers[idx][k]['floor'] = headers[idx][k].get('floor', pconfig.get('floor', None))
headers[idx][k]['minRange'] = headers[idx][k].get('minRange', pconfig.get('minRange', None))
headers[idx][k]['shared_key'] = headers[idx][k].get('shared_key', pconfig.get('shared_key', None))
headers[idx][k]['modify'] = headers[idx][k].get('modify', pconfig.get('modify', None))
headers[idx][k]['placement'] = float( headers[idx][k].get('placement', 1000) )
if headers[idx][k]['colour'] is None:
cidx = idx
while cidx >= len(sectcols):
cidx -= len(sectcols)
headers[idx][k]['colour'] = sectcols[cidx]
# Overwrite hidden if set in user config
for ns in config.table_columns_visible.keys():
# Make namespace key case insensitive
if ns.lower() == headers[idx][k]['namespace'].lower():
try:
# Config has True = visibile, False = Hidden. Here we're setting "hidden" which is inverse
headers[idx][k]['hidden'] = not config.table_columns_visible[ns][k]
except KeyError:
pass
# Also overwite placement if set in config
try:
headers[idx][k]['placement'] = float(config.table_columns_placement[ headers[idx][k]['namespace'] ][k])
except (KeyError, ValueError):
try:
headers[idx][k]['placement'] = float(config.table_columns_placement[ pconfig['id'] ][k])
except (KeyError, ValueError):
pass
# Work out max and min value if not given
setdmax = False
setdmin = False
try:
headers[idx][k]['dmax'] = float(headers[idx][k]['max'])
except TypeError:
headers[idx][k]['dmax'] = 0
setdmax = True
try:
headers[idx][k]['dmin'] = float(headers[idx][k]['min'])
except TypeError:
headers[idx][k]['dmin'] = 0
setdmin = True
# Figure out the min / max if not supplied
if setdmax or setdmin:
for s_name, samp in data[idx].items():
try:
val = float(samp[k])
if callable(headers[idx][k]['modify']):
val = float(headers[idx][k]['modify'](val))
if setdmax:
headers[idx][k]['dmax'] = max(headers[idx][k]['dmax'], val)
if setdmin:
headers[idx][k]['dmin'] = min(headers[idx][k]['dmin'], val)
except ValueError:
val = samp[k] # couldn't convert to float - keep as a string
except KeyError:
pass # missing data - skip
# Limit auto-generated scales with floor, ceiling and minRange.
if headers[idx][k]['ceiling'] is not None and headers[idx][k]['max'] is None:
headers[idx][k]['dmax'] = min(headers[idx][k]['dmax'], float(headers[idx][k]['ceiling']))
if headers[idx][k]['floor'] is not None and headers[idx][k]['min'] is None:
headers[idx][k]['dmin'] = max(headers[idx][k]['dmin'], float(headers[idx][k]['floor']))
if headers[idx][k]['minRange'] is not None:
drange = headers[idx][k]['dmax'] - headers[idx][k]['dmin']
if drange < float(headers[idx][k]['minRange']):
headers[idx][k]['dmax'] = headers[idx][k]['dmin'] + float(headers[idx][k]['minRange'])
# Collect settings for shared keys
shared_keys = defaultdict(lambda: dict())
for idx, hs in enumerate(headers):
for k in hs.keys():
sk = headers[idx][k]['shared_key']
if sk is not None:
shared_keys[sk]['dmax'] = max(headers[idx][k]['dmax'], shared_keys[sk].get('dmax', headers[idx][k]['dmax']))
shared_keys[sk]['dmin'] = max(headers[idx][k]['dmin'], shared_keys[sk].get('dmin', headers[idx][k]['dmin']))
# Overwrite shared key settings and at the same time assign to buckets for sorting
# Within each section of headers, sort explicitly by 'title' if the dict
# is not already ordered, so the final ordering is by:
# placement > section > explicit_ordering > title
# Of course, the user can shuffle these manually.
self.headers_in_order = defaultdict(list)
for idx, hs in enumerate(headers):
keys_in_section = hs.keys()
if type(hs) is not OrderedDict:
keys_in_section = sorted(keys_in_section, key=lambda k: headers[idx][k]['title'])
for k in keys_in_section:
sk = headers[idx][k]['shared_key']
if sk is not None:
headers[idx][k]['dmax'] = shared_keys[sk]['dmax']
headers[idx][k]['dmin'] = shared_keys[sk]['dmin']
self.headers_in_order[headers[idx][k]['placement']].append((idx, k))
# Assign to class
self.data = data
self.headers = headers
self.pconfig = pconfig
def make_plot(dt):
bs_id = dt.pconfig.get(
"id", "table_{}".format("".join(random.sample(letters, 4))))
# Sanitise plot ID and check for duplicates
bs_id = report.save_htmlid(bs_id)
categories = []
s_names = []
data = []
for idx, hs in enumerate(dt.headers):
for k, header in hs.items():
bcol = "rgb({})".format(header.get("colour", "204,204,204"))
categories.append({
"namespace": header["namespace"],
"title": header["title"],
"description": header["description"],
"max": header["dmax"],
"min": header["dmin"],
"suffix": header.get("suffix", ""),
"decimalPlaces": header.get("decimalPlaces", "2"),
"bordercol": bcol,
})
# Add the data
thisdata = []
these_snames = []
for (s_name, samp) in dt.data[idx].items():
if k in samp:
val = samp[k]
if "modify" in header and callable(header["modify"]):
val = header["modify"](val)
thisdata.append(val)
these_snames.append(s_name)
data.append(thisdata)
s_names.append(these_snames)
if len(s_names) == 0:
logger.warning("Tried to make beeswarm plot, but had no data")
return '<p class="text-danger">Error - was not able to plot data.</p>'
# Plot HTML
html = """<div class="hc-plot-wrapper">
<div id="{bid}" class="hc-plot not_rendered hc-beeswarm-plot"><small>loading..</small></div>
</div>""".format(bid=bs_id)
report.num_hc_plots += 1
report.plot_data[bs_id] = {
"plot_type": "beeswarm",
"samples": s_names,
"datasets": data,
"categories": categories
}
return html
def highcharts_bargraph (plotdata, plotsamples=None, pconfig=None):
"""
Build the HTML needed for a HighCharts bar graph. Should be
called by plot_bargraph, which properly formats input data.
"""
if pconfig is None:
pconfig = {}
if pconfig.get('id') is None:
pconfig['id'] = 'mqc_hcplot_'+''.join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig['id'] = report.save_htmlid(pconfig['id'])
html = '<div class="mqc_hcplot_plotgroup">'
# Counts / Percentages / Log Switches
if pconfig.get('cpswitch') is not False or pconfig.get('logswitch') is True:
if pconfig.get('cpswitch_c_active', True) is True:
c_active = 'active'
p_active = ''
l_active = ''
elif pconfig.get('logswitch_active') is True:
c_active = ''
p_active = ''
l_active = 'active'
else:
c_active = ''
p_active = 'active'
l_active = ''
pconfig['stacking'] = 'percent'
c_label = pconfig.get('cpswitch_counts_label', 'Counts')
p_label = pconfig.get('cpswitch_percent_label', 'Percentages')
l_label = pconfig.get('logswitch_label', 'Log10')
html += '<div class="btn-group hc_switch_group"> \n'
html += '<button class="btn btn-default btn-sm {c_a}" data-action="set_numbers" data-target="{id}" data-ylab="{c_l}">{c_l}</button> \n'.format(id=pconfig['id'], c_a=c_active, c_l=c_label)
if pconfig.get('cpswitch', True) is True:
html += '<button class="btn btn-default btn-sm {p_a}" data-action="set_percent" data-target="{id}" data-ylab="{p_l}">{p_l}</button> \n'.format(id=pconfig['id'], p_a=p_active, p_l=p_label)
if pconfig.get('logswitch') is True:
html += '<button class="btn btn-default btn-sm {l_a}" data-action="set_log" data-target="{id}" data-ylab="{l_l}">{l_l}</button> \n'.format(id=pconfig['id'], l_a=l_active, l_l=l_label)
pconfig['reversedStacks'] = True
html += '</div> '
if len(plotdata) > 1:
html += ' '
# Buttons to cycle through different datasets
if len(plotdata) > 1:
html += '<div class="btn-group hc_switch_group">\n'
for k, p in enumerate(plotdata):
active = 'active' if k == 0 else ''
try:
name = pconfig['data_labels'][k]['name']
except:
try:
name = pconfig['data_labels'][k]
except:
name = k+1
try:
ylab = 'data-ylab="{}"'.format(pconfig['data_labels'][k]['ylab'])
except:
ylab = 'data-ylab="{}"'.format(name) if name != k+1 else ''
try:
ymax = 'data-ymax="{}"'.format(pconfig['data_labels'][k]['ymax'])
except:
ymax = ''
html += '<button class="btn btn-default btn-sm {a}" data-action="set_data" {y} {ym} data-newdata="{k}" data-target="{id}">{n}</button>\n'.format(a=active, id=pconfig['id'], n=name, y=ylab, ym=ymax, k=k)
html += '</div>\n\n'
# Plot HTML
html += """<div class="hc-plot-wrapper">
<div id="{id}" class="hc-plot not_rendered hc-bar-plot"><small>loading..</small></div>
</div></div>""".format(id=pconfig['id']);
report.num_hc_plots += 1
report.plot_data[pconfig['id']] = {
'plot_type': 'bar_graph',
'samples': plotsamples,
'datasets': plotdata,
'config': pconfig
}
return html
def highcharts_linegraph(plotdata, pconfig=None):
"""
Build the HTML needed for a HighCharts line graph. Should be
called by linegraph.plot(), which properly formats input data.
"""
if pconfig is None:
pconfig = {}
# Get the plot ID
if pconfig.get('id') is None:
pconfig['id'] = 'mqc_hcplot_' + ''.join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig['id'] = report.save_htmlid(pconfig['id'])
# Build the HTML for the page
html = '<div class="mqc_hcplot_plotgroup">'
# Log Switch
if pconfig.get('logswitch') is True:
c_active = 'active'
l_active = ''
if pconfig.get('logswitch_active') is True:
c_active = ''
l_active = 'active'
c_label = pconfig.get('cpswitch_counts_label', 'Counts')
l_label = pconfig.get('logswitch_label', 'Log10')
html += '<div class="btn-group hc_switch_group"> \n'
html += '<button class="btn btn-default btn-sm {c_a}" data-action="set_numbers" data-target="{id}" data-ylab="{c_l}">{c_l}</button> \n'.format(
id=pconfig['id'], c_a=c_active, c_l=c_label)
if pconfig.get('logswitch') is True:
html += '<button class="btn btn-default btn-sm {l_a}" data-action="set_log" data-target="{id}" data-ylab="{l_l}">{l_l}</button> \n'.format(
id=pconfig['id'], l_a=l_active, l_l=l_label)
html += '</div> '
if len(plotdata) > 1:
html += ' '
# Buttons to cycle through different datasets
if len(plotdata) > 1:
html += '<div class="btn-group hc_switch_group">\n'
for k, p in enumerate(plotdata):
active = 'active' if k == 0 else ''
try:
name = pconfig['data_labels'][k]['name']
except:
name = k + 1
try:
ylab = 'data-ylab="{}"'.format(
pconfig['data_labels'][k]['ylab'])
except:
ylab = 'data-ylab="{}"'.format(name) if name != k + 1 else ''
try:
ymax = 'data-ymax="{}"'.format(
pconfig['data_labels'][k]['ymax'])
except:
ymax = ''
try:
xlab = 'data-xlab="{}"'.format(
pconfig['data_labels'][k]['xlab'])
except:
xlab = ''
html += '<button class="btn btn-default btn-sm {a}" data-action="set_data" {y} {ym} {x} data-newdata="{k}" data-target="{id}">{n}</button>\n'.format(
a=active,
id=pconfig['id'],
n=name,
y=ylab,
ym=ymax,
x=xlab,
k=k)
html += '</div>\n\n'
# The plot div
html += '<div class="hc-plot-wrapper"><div id="{id}" class="hc-plot not_rendered hc-line-plot"><small>loading..</small></div></div></div> \n'.format(
id=pconfig['id'])
report.num_hc_plots += 1
report.plot_data[pconfig['id']] = {
'plot_type': "xy_line",
'datasets': plotdata,
'config': pconfig
}
return html
def highcharts_linegraph(plotdata, pconfig=None):
"""
Build the HTML needed for a HighCharts line graph. Should be
called by linegraph.plot(), which properly formats input data.
"""
if pconfig is None:
pconfig = {}
# Get the plot ID
if pconfig.get("id") is None:
pconfig["id"] = "mqc_hcplot_" + "".join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig["id"] = report.save_htmlid(pconfig["id"])
# Build the HTML for the page
html = '<div class="mqc_hcplot_plotgroup">'
# Log Switch
if pconfig.get("logswitch") is True:
c_active = "active"
l_active = ""
if pconfig.get("logswitch_active") is True:
c_active = ""
l_active = "active"
c_label = pconfig.get("cpswitch_counts_label", "Counts")
l_label = pconfig.get("logswitch_label", "Log10")
html += '<div class="btn-group hc_switch_group"> \n'
html += '<button class="btn btn-default btn-sm {c_a}" data-action="set_numbers" data-target="{id}" data-ylab="{c_l}">{c_l}</button> \n'.format(
id=pconfig["id"], c_a=c_active, c_l=c_label)
if pconfig.get("logswitch") is True:
html += '<button class="btn btn-default btn-sm {l_a}" data-action="set_log" data-target="{id}" data-ylab="{l_l}">{l_l}</button> \n'.format(
id=pconfig["id"], l_a=l_active, l_l=l_label)
html += "</div> "
if len(plotdata) > 1:
html += " "
# Buttons to cycle through different datasets
if len(plotdata) > 1:
html += '<div class="btn-group hc_switch_group">\n'
for k, p in enumerate(plotdata):
active = "active" if k == 0 else ""
try:
name = pconfig["data_labels"][k]["name"]
except:
name = k + 1
try:
ylab = 'data-ylab="{}"'.format(
pconfig["data_labels"][k]["ylab"])
except:
ylab = 'data-ylab="{}"'.format(name) if name != k + 1 else ""
try:
ymax = 'data-ymax="{}"'.format(
pconfig["data_labels"][k]["ymax"])
except:
ymax = ""
try:
xlab = 'data-xlab="{}"'.format(
pconfig["data_labels"][k]["xlab"])
except:
xlab = ""
html += '<button class="btn btn-default btn-sm {a}" data-action="set_data" {y} {ym} {x} data-newdata="{k}" data-target="{id}">{n}</button>\n'.format(
a=active,
id=pconfig["id"],
n=name,
y=ylab,
ym=ymax,
x=xlab,
k=k)
html += "</div>\n\n"
# The plot div
html += '<div class="hc-plot-wrapper"><div id="{id}" class="hc-plot not_rendered hc-line-plot"><small>loading..</small></div></div></div> \n'.format(
id=pconfig["id"])
report.num_hc_plots += 1
report.plot_data[pconfig["id"]] = {
"plot_type": "xy_line",
"datasets": plotdata,
"config": pconfig
}
return html
def make_plot(dt):
bs_id = dt.pconfig.get('id', 'table_{}'.format(''.join(random.sample(letters, 4))) )
# Sanitise plot ID and check for duplicates
bs_id = report.save_htmlid(bs_id)
categories = []
s_names = []
data = []
for idx, hs in enumerate(dt.headers):
for k, header in hs.items():
bcol = 'rgb({})'.format(header.get('colour', '204,204,204'))
categories.append({
'namespace': header['namespace'],
'title': header['title'],
'description': header['description'],
'max': header['dmax'],
'min': header['dmin'],
'suffix': header.get('suffix', ''),
'decimalPlaces': header.get('decimalPlaces', '2'),
'bordercol': bcol
});
# Add the data
thisdata = []
these_snames = []
for (s_name, samp) in dt.data[idx].items():
if k in samp:
val = samp[k]
if 'modify' in header and callable(header['modify']):
val = header['modify'](val)
thisdata.append(val)
these_snames.append(s_name)
data.append(thisdata)
s_names.append(these_snames)
if len(s_names) == 0:
logger.warning('Tried to make beeswarm plot, but had no data')
return '<p class="text-danger">Error - was not able to plot data.</p>'
# Plot HTML
html = """<div class="hc-plot-wrapper">
<div id="{bid}" class="hc-plot not_rendered hc-beeswarm-plot"><small>loading..</small></div>
</div>""".format(bid=bs_id)
report.num_hc_plots += 1
report.plot_data[bs_id] = {
'plot_type': 'beeswarm',
'samples': s_names,
'datasets': data,
'categories': categories
}
return html
def __init__(
self,
name="base",
anchor="base",
target=None,
href=None,
info=None,
comment=None,
extra=None,
autoformat=True,
autoformat_type="markdown",
doi=[],
):
# Custom options from user config that can overwrite base module values
mod_cust_config = getattr(self, "mod_cust_config", {})
self.name = mod_cust_config.get("name", name)
self.anchor = mod_cust_config.get("anchor", anchor)
target = mod_cust_config.get("target", target)
self.href = mod_cust_config.get("href", href)
self.info = mod_cust_config.get("info", info)
self.comment = mod_cust_config.get("comment", comment)
self.extra = mod_cust_config.get("extra", extra)
self.doi = mod_cust_config.get("doi", doi)
# Specific module level config to overwrite (e.g. config.bcftools, config.fastqc)
config.update({anchor: mod_cust_config.get("custom_config", {})})
# Sanitise anchor ID and check for duplicates
self.anchor = report.save_htmlid(self.anchor)
# See if we have a user comment in the config
if self.anchor in config.section_comments:
self.comment = config.section_comments[self.anchor]
if self.info is None:
self.info = ""
# Always finish with a ".", as we may add a DOI after the intro.
if len(self.info) > 0 and self.info[-1] != ".":
self.info += "."
if self.extra is None:
self.extra = ""
self.doi_link = ""
if type(self.doi) is str:
self.doi = [self.doi]
if len(self.doi) > 0:
doi_links = []
for doi in self.doi:
# Build the HTML link for the DOI
doi_links.append(
f' <a class="module-doi" data-doi="{doi}" data-toggle="popover" href="https://doi.org/{doi}" target="_blank">{doi}</a>'
)
self.doi_link = '<em class="text-muted small" style="margin-left: 1rem;">DOI: {}.</em>'.format(
"; ".join(doi_links))
if target is None:
target = self.name
if self.href is not None:
self.mname = '<a href="{}" target="_blank">{}</a>'.format(
self.href, target)
else:
self.mname = target
if self.href or self.info or self.extra or self.doi_link:
self.intro = "<p>{} {}{}</p>{}".format(self.mname, self.info,
self.doi_link, self.extra)
# Format the markdown strings
if autoformat:
if self.comment is not None:
self.comment = textwrap.dedent(self.comment)
if autoformat_type == "markdown":
self.comment = markdown.markdown(self.comment)
self.sections = list()
def matplotlib_linegraph (plotdata, pconfig=None):
"""
Plot a line graph with Matplot lib and return a HTML string. Either embeds a base64
encoded image within HTML or writes the plot and links to it. Should be called by
plot_bargraph, which properly formats the input data.
"""
if pconfig is None:
pconfig = {}
# Plot group ID
if pconfig.get('id') is None:
pconfig['id'] = 'mqc_mplplot_'+''.join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig['id'] = report.save_htmlid(pconfig['id'])
# Individual plot IDs
pids = []
for k in range(len(plotdata)):
try:
name = pconfig['data_labels'][k]['name']
except:
name = k+1
pid = 'mqc_{}_{}'.format(pconfig['id'], name)
pid = report.save_htmlid(pid)
pids.append(pid)
html = '<p class="text-info"><small><span class="glyphicon glyphicon-picture" aria-hidden="true"></span> ' + \
'Flat image plot. Toolbox functions such as highlighting / hiding samples will not work ' + \
'(see the <a href="http://multiqc.info/docs/#flat--interactive-plots" target="_blank">docs</a>).</small></p>'
html += '<div class="mqc_mplplot_plotgroup" id="{}">'.format(pconfig['id'])
# Same defaults as HighCharts for consistency
default_colors = ['#7cb5ec', '#434348', '#90ed7d', '#f7a35c', '#8085e9',
'#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1']
# Buttons to cycle through different datasets
if len(plotdata) > 1 and not config.simple_output:
html += '<div class="btn-group mpl_switch_group mqc_mplplot_bargraph_switchds">\n'
for k, p in enumerate(plotdata):
pid = pids[k]
active = 'active' if k == 0 else ''
try:
name = pconfig['data_labels'][k]['name']
except:
name = k+1
html += '<button class="btn btn-default btn-sm {a}" data-target="#{pid}">{n}</button>\n'.format(a=active, pid=pid, n=name)
html += '</div>\n\n'
# Go through datasets creating plots
for pidx, pdata in enumerate(plotdata):
# Plot ID
pid = pids[pidx]
# Save plot data to file
fdata = OrderedDict()
lastcats = None
sharedcats = True
for d in pdata:
fdata[d['name']] = OrderedDict()
for i, x in enumerate(d['data']):
if type(x) is list:
fdata[d['name']][str(x[0])] = x[1]
# Check to see if all categories are the same
if lastcats is None:
lastcats = [x[0] for x in d['data']]
elif lastcats != [x[0] for x in d['data']]:
sharedcats = False
else:
try:
fdata[d['name']][pconfig['categories'][i]] = x
except (KeyError, IndexError):
fdata[d['name']][str(i)] = x
# Custom tsv output if the x axis varies
if not sharedcats and config.data_format == 'tsv':
fout = ''
for d in pdata:
fout += "\t"+"\t".join([str(x[0]) for x in d['data']])
fout += "\n{}\t".format(d['name'])
fout += "\t".join([str(x[1]) for x in d['data']])
fout += "\n"
with io.open (os.path.join(config.data_dir, '{}.txt'.format(pid)), 'w', encoding='utf-8') as f:
print( fout.encode('utf-8', 'ignore').decode('utf-8'), file=f )
else:
util_functions.write_data_file(fdata, pid)
# Set up figure
fig = plt.figure(figsize=(14, 6), frameon=False)
axes = fig.add_subplot(111)
# Go through data series
for idx, d in enumerate(pdata):
# Default colour index
cidx = idx
while cidx >= len(default_colors):
cidx -= len(default_colors)
# Line style
linestyle = 'solid'
if d.get('dashStyle', None) == 'Dash':
linestyle = 'dashed'
# Reformat data (again)
try:
axes.plot([x[0] for x in d['data']], [x[1] for x in d['data']], label=d['name'], color=d.get('color', default_colors[cidx]), linestyle=linestyle, linewidth=1, marker=None)
except TypeError:
# Categorical data on x axis
axes.plot(d['data'], label=d['name'], color=d.get('color', default_colors[cidx]), linewidth=1, marker=None)
# Tidy up axes
axes.tick_params(labelsize=8, direction='out', left=False, right=False, top=False, bottom=False)
axes.set_xlabel(pconfig.get('xlab', ''))
axes.set_ylabel(pconfig.get('ylab', ''))
# Dataset specific y label
try:
axes.set_ylabel(pconfig['data_labels'][pidx]['ylab'])
except:
pass
# Axis limits
default_ylimits = axes.get_ylim()
ymin = default_ylimits[0]
if 'ymin' in pconfig:
ymin = pconfig['ymin']
elif 'yCeiling' in pconfig:
ymin = min(pconfig['yCeiling'], default_ylimits[0])
ymax = default_ylimits[1]
if 'ymax' in pconfig:
ymax = pconfig['ymax']
elif 'yFloor' in pconfig:
ymax = max(pconfig['yCeiling'], default_ylimits[1])
if (ymax - ymin) < pconfig.get('yMinRange', 0):
ymax = ymin + pconfig['yMinRange']
axes.set_ylim((ymin, ymax))
# Dataset specific ymax
try:
axes.set_ylim((ymin, pconfig['data_labels'][pidx]['ymax']))
except:
pass
default_xlimits = axes.get_xlim()
xmin = default_xlimits[0]
if 'xmin' in pconfig:
xmin = pconfig['xmin']
elif 'xCeiling' in pconfig:
xmin = min(pconfig['xCeiling'], default_xlimits[0])
xmax = default_xlimits[1]
if 'xmax' in pconfig:
xmax = pconfig['xmax']
elif 'xFloor' in pconfig:
xmax = max(pconfig['xCeiling'], default_xlimits[1])
if (xmax - xmin) < pconfig.get('xMinRange', 0):
xmax = xmin + pconfig['xMinRange']
axes.set_xlim((xmin, xmax))
# Plot title
if 'title' in pconfig:
plt.text(0.5, 1.05, pconfig['title'], horizontalalignment='center', fontsize=16, transform=axes.transAxes)
axes.grid(True, zorder=10, which='both', axis='y', linestyle='-', color='#dedede', linewidth=1)
# X axis categories, if specified
if 'categories' in pconfig:
axes.set_xticks([i for i,v in enumerate(pconfig['categories'])])
axes.set_xticklabels(pconfig['categories'])
# Axis lines
xlim = axes.get_xlim()
axes.plot([xlim[0], xlim[1]], [0, 0], linestyle='-', color='#dedede', linewidth=2)
axes.set_axisbelow(True)
axes.spines['right'].set_visible(False)
axes.spines['top'].set_visible(False)
axes.spines['bottom'].set_visible(False)
axes.spines['left'].set_visible(False)
# Background colours, if specified
if 'yPlotBands' in pconfig:
xlim = axes.get_xlim()
for pb in pconfig['yPlotBands']:
axes.barh(pb['from'], xlim[1], height = pb['to']-pb['from'], left=xlim[0], color=pb['color'], linewidth=0, zorder=0)
if 'xPlotBands' in pconfig:
ylim = axes.get_ylim()
for pb in pconfig['xPlotBands']:
axes.bar(pb['from'], ylim[1], width = pb['to']-pb['from'], bottom=ylim[0], color=pb['color'], linewidth=0, zorder=0)
# Tight layout - makes sure that legend fits in and stuff
if len(pdata) <= 15:
axes.legend(loc='lower center', bbox_to_anchor=(0, -0.22, 1, .102), ncol=5, mode='expand', fontsize=8, frameon=False)
plt.tight_layout(rect=[0,0.08,1,0.92])
else:
plt.tight_layout(rect=[0,0,1,0.92])
# Should this plot be hidden on report load?
hidediv = ''
if pidx > 0:
hidediv = ' style="display:none;"'
# Save the plot to the data directory if export is requests
if config.export_plots:
for fformat in config.export_plot_formats:
# Make the directory if it doesn't already exist
plot_dir = os.path.join(config.plots_dir, fformat)
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
# Save the plot
plot_fn = os.path.join(plot_dir, '{}.{}'.format(pid, fformat))
fig.savefig(plot_fn, format=fformat, bbox_inches='tight')
# Output the figure to a base64 encoded string
if getattr(get_template_mod(), 'base64_plots', True) is True:
img_buffer = io.BytesIO()
fig.savefig(img_buffer, format='png', bbox_inches='tight')
b64_img = base64.b64encode(img_buffer.getvalue()).decode('utf8')
img_buffer.close()
html += '<div class="mqc_mplplot" id="{}"{}><img src="data:image/png;base64,{}" /></div>'.format(pid, hidediv, b64_img)
# Save to a file and link <img>
else:
plot_relpath = os.path.join(config.plots_dir_name, 'png', '{}.png'.format(pid))
html += '<div class="mqc_mplplot" id="{}"{}><img src="{}" /></div>'.format(pid, hidediv, plot_relpath)
plt.close(fig)
# Close wrapping div
html += '</div>'
report.num_mpl_plots += 1
return html
def multiqc(analysis_dir, dirs, dirs_depth, no_clean_sname, title,
report_comment, template, module_tag, module, exclude, outdir,
ignore, ignore_samples, sample_names, file_list, filename,
make_data_dir, no_data_dir, data_format, zip_data_dir, force,
ignore_symlinks, export_plots, plots_flat, plots_interactive, lint,
make_pdf, no_megaqc_upload, config_file, cl_config, verbose, quiet,
**kwargs):
"""MultiQC aggregates results from bioinformatics analyses across many samples into a single report.
It searches a given directory for analysis logs and compiles a HTML report.
It's a general use tool, perfect for summarising the output from numerous
bioinformatics tools.
To run, supply with one or more directory to scan for analysis results.
To run here, use 'multiqc .'
See http://multiqc.info for more details.
Author: Phil Ewels (http://phil.ewels.co.uk)
"""
# Set up logging level
loglevel = log.LEVELS.get(min(verbose, 1), "INFO")
if quiet:
loglevel = 'WARNING'
log.init_log(logger, loglevel=loglevel)
# Load config files
plugin_hooks.mqc_trigger('before_config')
config.mqc_load_userconfig(config_file)
plugin_hooks.mqc_trigger('config_loaded')
# Command-line config YAML
if len(cl_config) > 0:
config.mqc_cl_config(cl_config)
# Log the command used to launch MultiQC
report.multiqc_command = " ".join(sys.argv)
logger.debug("Command used: {}".format(report.multiqc_command))
# Check that we're running the latest version of MultiQC
if config.no_version_check is not True:
try:
response = urlopen('http://multiqc.info/version.php?v={}'.format(
config.short_version),
timeout=5)
remote_version = response.read().decode('utf-8').strip()
if version.StrictVersion(re.sub(
'[^0-9\.]', '', remote_version)) > version.StrictVersion(
re.sub('[^0-9\.]', '', config.short_version)):
logger.warn(
'MultiQC Version {} now available!'.format(remote_version))
else:
logger.debug(
'Latest MultiQC version is {}'.format(remote_version))
except Exception as e:
logger.debug(
'Could not connect to multiqc.info for version check: {}'.
format(e))
# Set up key variables (overwrite config vars from command line)
if template is not None:
config.template = template
if title is not None:
config.title = title
if report_comment is not None:
config.report_comment = report_comment
if dirs is True:
config.prepend_dirs = dirs
if dirs_depth is not None:
config.prepend_dirs = True
config.prepend_dirs_depth = dirs_depth
config.analysis_dir = analysis_dir
if outdir is not None:
config.output_dir = outdir
if no_clean_sname:
config.fn_clean_sample_names = False
logger.info("Not cleaning sample names")
if make_data_dir:
config.make_data_dir = True
if no_data_dir:
config.make_data_dir = False
if force:
config.force = True
if ignore_symlinks:
config.ignore_symlinks = True
if zip_data_dir:
config.zip_data_dir = True
if data_format is not None:
config.data_format = data_format
if export_plots:
config.export_plots = True
if plots_flat:
config.plots_force_flat = True
if plots_interactive:
config.plots_force_interactive = True
if lint:
config.lint = True
lint_helpers.run_tests()
if make_pdf:
config.template = 'simple'
if no_megaqc_upload:
config.megaqc_upload = False
else:
config.megaqc_upload = True
if sample_names:
config.load_sample_names(sample_names)
if module_tag is not None:
config.module_tag = module_tag
config.kwargs = kwargs # Plugin command line options
plugin_hooks.mqc_trigger('execution_start')
logger.info("This is MultiQC v{}".format(__version__))
logger.debug("Command : {}".format(' '.join(sys.argv)))
logger.debug("Working dir : {}".format(os.getcwd()))
if make_pdf:
logger.info('--pdf specified. Using non-interactive HTML template.')
logger.info("Template : {}".format(config.template))
if lint:
logger.info('--lint specified. Being strict with validation.')
# Add files if --file-list option is given
if file_list:
if len(analysis_dir) > 1:
raise ValueError(
"If --file-list is giving, analysis_dir should have only one plain text file."
)
config.analysis_dir = []
with (open(analysis_dir[0])) as in_handle:
for line in in_handle:
if os.path.exists(line.strip()):
path = os.path.abspath(line.strip())
config.analysis_dir.append(path)
if len(config.analysis_dir) == 0:
logger.error(
"No files or directories were added from {} using --file-list option."
.format(analysis_dir[0]))
logger.error(
"Please, check that {} contains correct paths.".format(
analysis_dir[0]))
raise ValueError("Any files or directories to be searched.")
if len(ignore) > 0:
logger.debug(
"Ignoring files, directories and paths that match: {}".format(
", ".join(ignore)))
config.fn_ignore_files.extend(ignore)
config.fn_ignore_dirs.extend(ignore)
config.fn_ignore_paths.extend(ignore)
if len(ignore_samples) > 0:
logger.debug("Ignoring sample names that match: {}".format(
", ".join(ignore_samples)))
config.sample_names_ignore.extend(ignore_samples)
if filename == 'stdout':
config.output_fn = sys.stdout
logger.info("Printing report to stdout")
else:
if title is not None and filename is None:
filename = re.sub('[^\w\.-]', '', re.sub('[-\s]+', '-',
title)).strip()
filename += '_multiqc_report'
if filename is not None:
if filename.endswith('.html'):
filename = filename[:-5]
config.output_fn_name = filename
config.data_dir_name = '{}_data'.format(filename)
if not config.output_fn_name.endswith('.html'):
config.output_fn_name = '{}.html'.format(config.output_fn_name)
# Print some status updates
if config.title is not None:
logger.info("Report title: {}".format(config.title))
if dirs:
logger.info("Prepending directory to sample names")
for d in config.analysis_dir:
logger.info("Searching '{}'".format(d))
# Prep module configs
config.top_modules = [
m if type(m) is dict else {
m: {}
} for m in config.top_modules
]
config.module_order = [
m if type(m) is dict else {
m: {}
} for m in config.module_order
]
mod_keys = [list(m.keys())[0] for m in config.module_order]
# Lint the module configs
if config.lint:
for m in config.avail_modules.keys():
if m not in mod_keys:
errmsg = "LINT: Module '{}' not found in config.module_order".format(
m)
logger.error(errmsg)
report.lint_errors.append(errmsg)
else:
for mo in config.module_order:
if m != 'custom_content' and m in mo.keys(
) and 'module_tag' not in mo[m]:
errmsg = "LINT: Module '{}' in config.module_order did not have 'module_tag' config".format(
m)
logger.error(errmsg)
report.lint_errors.append(errmsg)
# Get the avaiable tags to decide which modules to run.
modules_from_tags = set()
if config.module_tag is not None:
tags = config.module_tag
for m in config.module_order:
module_name = list(m.keys())[0] # only one name in each dict
for tag in tags:
for t in m[module_name].get('module_tag', []):
if tag.lower() == t.lower():
modules_from_tags.add(module_name)
# Get the list of modules we want to run, in the order that we want them
run_modules = [
m for m in config.top_modules
if list(m.keys())[0] in config.avail_modules.keys()
]
run_modules.extend([{
m: {}
} for m in config.avail_modules.keys()
if m not in mod_keys and m not in run_modules])
run_modules.extend([
m for m in config.module_order
if list(m.keys())[0] in config.avail_modules.keys() and list(m.keys())
[0] not in [list(rm.keys())[0] for rm in run_modules]
])
if module:
run_modules = [m for m in run_modules if list(m.keys())[0] in module]
logger.info('Only using modules {}'.format(', '.join(module)))
elif modules_from_tags:
run_modules = [
m for m in run_modules if list(m.keys())[0] in modules_from_tags
]
logger.info("Only using modules with '{}' tag".format(
', '.join(module_tag)))
if exclude:
logger.info("Excluding modules '{}'".format("', '".join(exclude)))
if 'general_stats' in exclude:
config.skip_generalstats = True
exclude = tuple(x for x in exclude if x != 'general_stats')
run_modules = [
m for m in run_modules if list(m.keys())[0] not in exclude
]
if len(run_modules) == 0:
logger.critical('No analysis modules specified!')
sys.exit(1)
run_module_names = [list(m.keys())[0] for m in run_modules]
logger.debug("Analysing modules: {}".format(', '.join(run_module_names)))
# Create the temporary working directories
tmp_dir = tempfile.mkdtemp()
logger.debug(
'Using temporary directory for creating report: {}'.format(tmp_dir))
config.data_tmp_dir = os.path.join(tmp_dir, 'multiqc_data')
if filename != 'stdout' and config.make_data_dir == True:
config.data_dir = config.data_tmp_dir
os.makedirs(config.data_dir)
else:
config.data_dir = None
config.plots_tmp_dir = os.path.join(tmp_dir, 'multiqc_plots')
if filename != 'stdout' and config.export_plots == True:
config.plots_dir = config.plots_tmp_dir
os.makedirs(config.plots_dir)
# Load the template
template_mod = config.avail_templates[config.template].load()
# Add an output subdirectory if specified by template
try:
config.output_dir = os.path.join(config.output_dir,
template_mod.output_subdir)
except AttributeError:
pass # No subdirectory variable given
# Add custom content section names
try:
if 'custom_content' in run_module_names:
run_module_names.extend(config.custom_data.keys())
except AttributeError:
pass # custom_data not in config
# Get the list of files to search
report.get_filelist(run_module_names)
# Run the modules!
plugin_hooks.mqc_trigger('before_modules')
report.modules_output = list()
sys_exit_code = 0
for mod_dict in run_modules:
try:
this_module = list(mod_dict.keys())[0]
mod_cust_config = list(mod_dict.values())[0]
mod = config.avail_modules[this_module].load()
mod.mod_cust_config = mod_cust_config # feels bad doing this, but seems to work
output = mod()
if type(output) != list:
output = [output]
for m in output:
report.modules_output.append(m)
# Copy over css & js files if requested by the theme
try:
for to, path in report.modules_output[-1].css.items():
copy_to = os.path.join(tmp_dir, to)
os.makedirs(os.path.dirname(copy_to))
shutil.copyfile(path, copy_to)
except OSError as e:
if e.errno == errno.EEXIST:
pass
else:
raise
except AttributeError:
pass
try:
for to, path in report.modules_output[-1].js.items():
copy_to = os.path.join(tmp_dir, to)
os.makedirs(os.path.dirname(copy_to))
shutil.copyfile(path, copy_to)
except OSError as e:
if e.errno == errno.EEXIST:
pass
else:
raise
except AttributeError:
pass
except UserWarning:
logger.debug("No samples found: {}".format(
list(mod_dict.keys())[0]))
except KeyboardInterrupt:
shutil.rmtree(tmp_dir)
logger.critical("User Cancelled Execution!\n{eq}\n{tb}{eq}\n".
format(eq=('=' * 60), tb=traceback.format_exc()) +
"User Cancelled Execution!\nExiting MultiQC...")
sys.exit(1)
except:
# Flag the error, but carry on
logger.error("Oops! The '{}' MultiQC module broke... \n".format(this_module) + \
" Please copy the following traceback and report it at " + \
"https://github.com/ewels/MultiQC/issues \n" + \
" If possible, please include a log file that triggers the error - " + \
"the last file found was:\n" + \
" {}\n".format(report.last_found_file) + \
('='*60)+"\nModule {} raised an exception: {}".format(
this_module, traceback.format_exc()) + ('='*60))
sys_exit_code = 1
# Did we find anything?
if len(report.modules_output) == 0:
logger.warn("No analysis results found. Cleaning up..")
shutil.rmtree(tmp_dir)
logger.info("MultiQC complete")
# Exit with an error code if a module broke
sys.exit(sys_exit_code)
# Sort the report sections if we have a config
if len(getattr(config, 'report_section_order', {})) > 0:
section_id_order = {}
idx = 10
for mod in reversed(report.modules_output):
section_id_order[mod.anchor] = idx
idx += 10
for anchor, ss in config.report_section_order.items():
if anchor not in section_id_order.keys():
continue
if ss.get('order') is not None:
section_id_order[anchor] = ss['order']
if ss.get('after') in section_id_order.keys():
section_id_order[anchor] = section_id_order[ss['after']] + 1
if ss.get('before') in section_id_order.keys():
section_id_order[anchor] = section_id_order[ss['before']] - 1
sorted_ids = sorted(section_id_order, key=section_id_order.get)
report.modules_output = [
mod for i in reversed(sorted_ids) for mod in report.modules_output
if mod.anchor == i
]
plugin_hooks.mqc_trigger('after_modules')
# Remove empty data sections from the General Stats table
empty_keys = [
i for i, d in enumerate(report.general_stats_data[:]) if len(d) == 0
]
empty_keys.sort(reverse=True)
for i in empty_keys:
del report.general_stats_data[i]
del report.general_stats_headers[i]
# Add general-stats IDs to table row headers
for idx, h in enumerate(report.general_stats_headers):
for k in h.keys():
if 'rid' not in h[k]:
h[k]['rid'] = re.sub(r'\W+', '_', k).strip().strip('_')
ns_html = re.sub(r'\W+', '_',
h[k]['namespace']).strip().strip('_').lower()
report.general_stats_headers[idx][k]['rid'] = report.save_htmlid(
'mqc-generalstats-{}-{}'.format(ns_html, h[k]['rid']))
# Generate the General Statistics HTML & write to file
if len(report.general_stats_data) > 0:
pconfig = {
'id': 'general_stats_table',
'table_title': 'General Statistics',
'save_file': True,
'raw_data_fn': 'multiqc_general_stats'
}
report.general_stats_html = table.plot(report.general_stats_data,
report.general_stats_headers,
pconfig)
else:
config.skip_generalstats = True
# Write the report sources to disk
if config.data_dir is not None:
report.data_sources_tofile()
# Compress the report plot JSON data
logger.info("Compressing plot data")
report.plot_compressed_json = report.compress_json(report.plot_data)
plugin_hooks.mqc_trigger('before_report_generation')
# Data Export / MegaQC integration - save report data to file or send report data to an API endpoint
if (config.data_dump_file or config.megaqc_url) and config.megaqc_upload:
multiqc_json_dump = megaqc.multiqc_dump_json(report)
if config.data_dump_file:
util_functions.write_data_file(multiqc_json_dump, 'multiqc_data',
False, 'json')
if config.megaqc_url:
megaqc.multiqc_api_post(multiqc_json_dump)
# Make the final report path & data directories
if filename != 'stdout':
config.output_fn = os.path.join(config.output_dir,
config.output_fn_name)
config.data_dir = os.path.join(config.output_dir, config.data_dir_name)
# Check for existing reports and remove if -f was specified
if os.path.exists(
config.output_fn) or (config.make_data_dir
and os.path.exists(config.data_dir)):
if config.force:
if os.path.exists(config.output_fn):
logger.warning(
"Deleting : {} (-f was specified)".format(
os.path.relpath(config.output_fn)))
os.remove(config.output_fn)
if config.make_data_dir and os.path.exists(config.data_dir):
logger.warning(
"Deleting : {} (-f was specified)".format(
os.path.relpath(config.data_dir)))
shutil.rmtree(config.data_dir)
else:
# Set up the base names of the report and the data dir
report_num = 1
report_base, report_ext = os.path.splitext(
config.output_fn_name)
dir_base = os.path.basename(config.data_dir)
# Iterate through appended numbers until we find one that's free
while os.path.exists(
config.output_fn) or (config.make_data_dir and
os.path.exists(config.data_dir)):
config.output_fn = os.path.join(
config.output_dir,
"{}_{}{}".format(report_base, report_num, report_ext))
config.data_dir = os.path.join(
config.output_dir,
"{}_{}".format(dir_base, report_num))
report_num += 1
config.output_fn_name = os.path.basename(config.output_fn)
config.data_dir_name = os.path.basename(config.data_dir)
logger.warning(
"Previous MultiQC output found! Adjusting filenames..")
logger.warning(
"Use -f or --force to overwrite existing reports instead")
# Make directories for report if needed
if not os.path.exists(os.path.dirname(config.output_fn)):
os.makedirs(os.path.dirname(config.output_fn))
logger.info("Report : {}".format(os.path.relpath(
config.output_fn)))
if config.make_data_dir == False:
logger.info("Data : None")
else:
# Make directories for data_dir
logger.info("Data : {}".format(
os.path.relpath(config.data_dir)))
if not os.path.exists(config.data_dir):
os.makedirs(config.data_dir)
# Modules have run, so data directory should be complete by now. Move its contents.
for f in os.listdir(config.data_tmp_dir):
fn = os.path.join(config.data_tmp_dir, f)
logger.debug("Moving data file from '{}' to '{}'".format(
fn, config.data_dir))
shutil.move(fn, config.data_dir)
# Copy across the static plot images if requested
if config.export_plots:
config.plots_dir = os.path.join(config.output_dir,
config.plots_dir_name)
if os.path.exists(config.plots_dir):
if config.force:
logger.warning(
"Deleting : {} (-f was specified)".format(
os.path.relpath(config.plots_dir)))
shutil.rmtree(config.plots_dir)
else:
logger.error("Output directory {} already exists.".format(
config.plots_dir))
logger.info(
"Use -f or --force to overwrite existing reports")
shutil.rmtree(tmp_dir)
sys.exit(1)
os.makedirs(config.plots_dir)
logger.info("Plots : {}".format(
os.path.relpath(config.plots_dir)))
# Modules have run, so plots directory should be complete by now. Move its contents.
for f in os.listdir(config.plots_tmp_dir):
fn = os.path.join(config.plots_tmp_dir, f)
logger.debug("Moving plots directory from '{}' to '{}'".format(
fn, config.plots_dir))
shutil.move(fn, config.plots_dir)
plugin_hooks.mqc_trigger('before_template')
# Load in parent template files first if a child theme
try:
parent_template = config.avail_templates[
template_mod.template_parent].load()
copy_tree(parent_template.template_dir, tmp_dir)
except AttributeError:
pass # Not a child theme
# Copy the template files to the tmp directory (distutils overwrites parent theme files)
copy_tree(template_mod.template_dir, tmp_dir)
# Function to include file contents in Jinja template
def include_file(name, fdir=tmp_dir, b64=False):
try:
if fdir is None:
fdir = ''
if b64:
with io.open(os.path.join(fdir, name), "rb") as f:
return base64.b64encode(f.read()).decode('utf-8')
else:
with io.open(os.path.join(fdir, name), "r",
encoding='utf-8') as f:
return f.read()
except (OSError, IOError) as e:
logger.error("Could not include file '{}': {}".format(name, e))
# Load the report template
try:
env = jinja2.Environment(loader=jinja2.FileSystemLoader(tmp_dir))
env.globals['include_file'] = include_file
j_template = env.get_template(template_mod.base_fn)
except:
raise IOError("Could not load {} template file '{}'".format(
config.template, template_mod.base_fn))
# Use jinja2 to render the template and overwrite
config.analysis_dir = [os.path.realpath(d) for d in config.analysis_dir]
report_output = j_template.render(report=report, config=config)
if filename == 'stdout':
print(report_output.encode('utf-8'), file=sys.stdout)
else:
try:
with io.open(config.output_fn, "w", encoding='utf-8') as f:
print(report_output, file=f)
except IOError as e:
raise IOError("Could not print report to '{}' - {}".format(
config.output_fn, IOError(e)))
# Copy over files if requested by the theme
try:
for f in template_mod.copy_files:
fn = os.path.join(tmp_dir, f)
dest_dir = os.path.join(os.path.dirname(config.output_fn), f)
copy_tree(fn, dest_dir)
except AttributeError:
pass # No files to copy
# Clean up temporary directory
shutil.rmtree(tmp_dir)
# Zip the data directory if requested
if config.zip_data_dir and config.data_dir is not None:
shutil.make_archive(config.data_dir, 'zip', config.data_dir)
shutil.rmtree(config.data_dir)
# Try to create a PDF if requested
if make_pdf:
try:
pdf_fn_name = config.output_fn.replace('.html', '.pdf')
pandoc_call = [
'pandoc', '--standalone', config.output_fn, '--output',
pdf_fn_name, '--pdf-engine=xelatex', '-V',
'documentclass=article', '-V', 'geometry=margin=1in', '-V',
'title='
]
if config.pandoc_template is not None:
pandoc_call.append('--template={}'.format(
config.pandoc_template))
logger.debug(
"Attempting Pandoc conversion to PDF with following command:\n{}"
.format(' '.join(pandoc_call)))
pdf_exit_code = subprocess.call(pandoc_call)
if pdf_exit_code != 0:
logger.error(
"Error creating PDF! Pandoc returned a non-zero exit code."
)
else:
logger.info("PDF Report : {}".format(pdf_fn_name))
except OSError as e:
if e.errno == os.errno.ENOENT:
logger.error(
'Error creating PDF - pandoc not found. Is it installed? http://pandoc.org/'
)
else:
logger.error(
"Error creating PDF! Something went wrong when creating the PDF\n"
+ ('=' * 60) + "\n{}\n".format(traceback.format_exc()) +
('=' * 60))
plugin_hooks.mqc_trigger('execution_finish')
logger.info("MultiQC complete")
if lint and len(report.lint_errors) > 0:
logger.error("Found {} linting errors!\n{}".format(
len(report.lint_errors), "\n".join(report.lint_errors)))
sys_exit_code = 1
# Move the log file into the data directory
log.move_tmp_log(logger)
# Exit with an error code if a module broke
sys.exit(sys_exit_code)
def make_table(dt, hide_bar=None):
"""
Build the HTML needed for a MultiQC table.
:param data: MultiQC datatable object
"""
table_id = dt.pconfig.get(
'id', 'table_{}'.format(''.join(random.sample(letters, 4))))
table_id = report.save_htmlid(table_id)
t_headers = OrderedDict()
t_modal_headers = OrderedDict()
t_rows = OrderedDict()
t_rows_empty = OrderedDict()
dt.raw_vals = defaultdict(lambda: dict())
empty_cells = dict()
hidden_cols = 1
table_title = dt.pconfig.get('table_title')
if table_title is None:
table_title = table_id.replace("_", " ").title()
for idx, k, header in dt.get_headers_in_order():
rid = header['rid']
# Build the table header cell
shared_key = ''
if header.get('shared_key', None) is not None:
shared_key = ' data-shared-key={}'.format(header['shared_key'])
hide = ''
muted = ''
checked = ' checked="checked"'
if header.get('hidden', False) is True:
hide = 'hidden'
muted = ' text-muted'
checked = ''
hidden_cols += 1
data_attr = 'data-dmax="{}" data-dmin="{}" data-namespace="{}" {}' \
.format(header['dmax'], header['dmin'], header['namespace'], shared_key)
if header.get('namespace'):
cell_contents = '<span class="mqc_table_tooltip" title="{}: {}">{}</span>' \
.format(header['namespace'], header['description'], header['title'])
else:
cell_contents = '<span class="mqc_table_tooltip" title="{} {}">{}</span>' \
.format(header['namespace'], header['description'], header['title'])
t_headers[rid] = '<th id="header_{rid}" class="{rid} {h}" {da}>{c}</th>' \
.format(rid=rid, h=hide, da=data_attr, c=cell_contents)
empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format(
rid=rid, h=hide)
# Build the modal table row
t_modal_headers[rid] = """
<tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);">
<td class="sorthandle ui-sortable-handle">||</span></td>
<td style="text-align:center;">
<input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}">
</td>
<td>{name}</td>
<td>{title}</td>
<td>{desc}</td>
<td>{col_id}</td>
<td>{sk}</td>
</tr>""".format(rid=rid,
muted=muted,
checked=checked,
tid=table_id,
col=header['colour'],
name=header['namespace'],
title=header['title'],
desc=header['description'],
col_id='<code>{}</code>'.format(k),
sk=header.get('shared_key', ''))
# Make a colour scale
if header['scale'] == False:
c_scale = None
else:
c_scale = mqc_colour.mqc_colour_scale(header['scale'],
header['dmin'],
header['dmax'])
# Add the data table cells
for (s_name, samp) in dt.data[idx].items():
if k in samp:
val = samp[k]
kname = '{}_{}'.format(header['namespace'], rid)
dt.raw_vals[s_name][kname] = val
# if "is_int" in header.keys():
# val = int(val)
# print(val)
if 'modify' in header and callable(header['modify']):
val = header['modify'](val)
try:
dmin = header['dmin']
dmax = header['dmax']
percentage = ((float(val) - dmin) / (dmax - dmin)) * 100
percentage = min(percentage, 100)
percentage = max(percentage, 0)
except (ZeroDivisionError, ValueError):
percentage = 0
try:
if not header.get("is_int"):
valstring = str(header['format'].format(val))
else:
valstring = str(int(val))
except ValueError:
try:
valstring = str(header['format'].format(float(val)))
except ValueError:
valstring = str(val)
except:
valstring = str(val)
# This is horrible, but Python locale settings are worse
if config.thousandsSep_format is None:
config.thousandsSep_format = '<span class="mqc_thousandSep"></span>'
if config.decimalPoint_format is None:
config.decimalPoint_format = '.'
valstring = valstring.replace('.', 'DECIMAL').replace(
',', 'THOUSAND')
valstring = valstring.replace(
'DECIMAL', config.decimalPoint_format).replace(
'THOUSAND', config.thousandsSep_format)
# Percentage suffixes etc
if header.get('suffix') == "show_perc":
suff_dict = header.get("suffix_dict")
valstring += str(suff_dict.get(s_name))
else:
valstring += header.get('suffix', '')
# Conditional formatting
cmatches = {
cfck: False
for cfc in config.table_cond_formatting_colours
for cfck in cfc
}
# Find general rules followed by column-specific rules
for cfk in ['all_columns', rid]:
if cfk in config.table_cond_formatting_rules:
# Loop through match types
for ftype in cmatches.keys():
# Loop through array of comparison types
for cmp in config.table_cond_formatting_rules[
cfk].get(ftype, []):
try:
# Each comparison should be a dict with single key: val
if 's_eq' in cmp and str(
cmp['s_eq']).lower() == str(
val).lower():
cmatches[ftype] = True
if 's_contains' in cmp and str(
cmp['s_contains']).lower() in str(
val).lower():
cmatches[ftype] = True
if 's_ne' in cmp and str(
cmp['s_ne']).lower() != str(
val).lower():
cmatches[ftype] = True
if 'eq' in cmp and float(
cmp['eq']) == float(val):
cmatches[ftype] = True
if 'ne' in cmp and float(
cmp['ne']) != float(val):
cmatches[ftype] = True
if 'gt' in cmp and float(
cmp['gt']) < float(val):
cmatches[ftype] = True
if 'lt' in cmp and float(
cmp['lt']) > float(val):
cmatches[ftype] = True
except:
logger.warn(
"Not able to apply table conditional formatting to '{}' ({})"
.format(val, cmp))
# Apply HTML in order of config keys
bgcol = None
for cfc in config.table_cond_formatting_colours:
for cfck in cfc: # should always be one, but you never know
if cmatches[cfck]:
bgcol = cfc[cfck]
if bgcol is not None:
valstring = '<span class="badge" style="background-color:{}">{}</span>'.format(
bgcol, valstring)
# Build HTML
if not header['scale']:
if s_name not in t_rows:
t_rows[s_name] = dict()
t_rows[s_name][
rid] = '<td class="{rid} {h}">{v}</td>'.format(
rid=rid, h=hide, v=valstring)
# coloring with quartiles
elif header['scale'] == "quart":
col_dict = header['col_dict']
bar_dict = header.get('bar_dict')
col = ' background-color:{};'.format(col_dict.get(s_name))
if bar_dict:
bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(
bar_dict.get(s_name) + 3, col)
else:
bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(
percentage, col)
# bar percentage here
val_html = '<span class="val">{}</span>'.format(valstring)
wrapper_html = '<div class="wrapper">{}{}</div>'.format(
bar_html, val_html)
if s_name not in t_rows:
t_rows[s_name] = dict()
t_rows[s_name][
rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(
rid=rid, h=hide, c=wrapper_html)
else:
if c_scale is not None:
col = ' background-color:{};'.format(
c_scale.get_colour(val))
else:
col = ''
bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(
percentage, col)
val_html = '<span class="val">{}</span>'.format(valstring)
wrapper_html = '<div class="wrapper">{}{}</div>'.format(
bar_html, val_html)
if s_name not in t_rows:
t_rows[s_name] = dict()
t_rows[s_name][
rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(
rid=rid, h=hide, c=wrapper_html)
# Is this cell hidden or empty?
if s_name not in t_rows_empty:
t_rows_empty[s_name] = dict()
t_rows_empty[s_name][rid] = header.get(
'hidden', False) or str(val).strip() == ''
# Remove header if we don't have any filled cells for it
if sum([len(rows) for rows in t_rows.values()]) == 0:
t_headers.pop(rid, None)
t_modal_headers.pop(rid, None)
logger.debug(
'Removing header {} from general stats table, as no data'.
format(k))
#
# Put everything together
#
# Buttons above the table
html = ''
if not config.simple_output:
# Copy Table Button
html += """<div class="row">
<div class="col-sm-2">
<button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}">
<span class="glyphicon glyphicon-copy"></span> Copy table
</button>
</div>
""".format(tid=table_id)
# <div class="progress-bar progress-bar-warning progress-bar-striped" style="width: 20%">
# <span class="sr-only">20% Complete (warning)</span>
# </div>
# <div class="progress-bar progress-bar-danger" style="width: 10%">
# <span class="sr-only">10% Complete (danger)</span>
# </div>
if not hide_bar:
html += """ <div class="col-sm-2">
<div id="quartiles_bar">
<div class="progress">
<div class="progress-bar progress-bar-q1" style="width: 25%">
<span class="sr-only">35% Complete (success)</span> Q1
</div>
<div class="progress-bar progress-bar-q2" style="width: 25%">
<span class="sr-only">35% Complete (success)</span> Q2
</div>
<div class="progress-bar progress-bar-q3" style="width: 25%">
<span class="sr-only">35% Complete (success)</span> Q3
</div>
<div class="progress-bar progress-bar-q4" style="width: 25%">
<span class="sr-only">35% Complete (success)</span> Q4
</div>
</div>
</div>
</div>
</div>
"""
else:
html += """ </div>
"""
# Configure Columns Button
# if len(t_headers) > 1:
# html += """
# <button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal">
# <span class="glyphicon glyphicon-th"></span> Configure Columns
# </button>
# """.format(tid=table_id)
# Sort By Highlight button
# html += """
# <button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;">
# <span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight
# </button>
# """.format(tid=table_id)
# Scatter Plot Button
# if len(t_headers) > 1:
# html += """
# <button type="button" class="mqc_table_makeScatter btn btn-default btn-sm" data-toggle="modal" data-target="#tableScatterModal" data-table="#{tid}">
# <span class="glyphicon glyphicon glyphicon-stats"></span> Plot
# </button>
# """.format(tid=table_id)
# "Showing x of y columns" text
row_visibilities = [
all(t_rows_empty[s_name].values()) for s_name in t_rows_empty
]
visible_rows = [x for x in row_visibilities if not x]
# html += """
# <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nvisrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small>
# """.format(tid=table_id, nvisrows=len(visible_rows), nrows=len(t_rows), ncols_vis = (len(t_headers)+1)-hidden_cols, ncols=len(t_headers))
# Add text
# html += """
# <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nvisrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small>
# """.format(tid=table_id, nvisrows=len(visible_rows), nrows=len(t_rows),
# ncols_vis=(len(t_headers) + 1) - hidden_cols, ncols=len(t_headers))
# Build the table itself
collapse_class = 'mqc-table-collapse' if len(
t_rows) > 10 and config.collapse_tables else ''
html += """
<div id="{tid}_container" class="mqc_table_container">
<div class="table-responsive mqc-table-responsive {cc}">
<table id="{tid}" class="table table-condensed mqc_table" data-title="{title}">
""".format(tid=table_id, title=table_title, cc=collapse_class)
# Build the header row
col1_header = dt.pconfig.get('col1_header', 'Sample Name')
html += '<thead><tr><th class="rowheader">{}</th>{}</tr></thead>'.format(
col1_header, ''.join(t_headers.values()))
# Build the table body
html += '<tbody>'
t_row_keys = t_rows.keys()
if dt.pconfig.get('sortRows') is not False:
t_row_keys = sorted(t_row_keys)
for s_name in t_row_keys:
# Hide the row if all cells are empty or hidden
row_hidden = ' style="display:none"' if all(
t_rows_empty[s_name].values()) else ''
html += '<tr{}>'.format(row_hidden)
# Sample name row header
html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format(
sn=s_name)
for k in t_headers:
html += t_rows[s_name].get(k, empty_cells[k])
html += '</tr>'
html += '</tbody></table></div>'
if len(t_rows) > 10 and config.collapse_tables:
html += '<div class="mqc-table-expand"><span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span></div>'
html += '</div>'
# Build the bootstrap modal to customise columns and order
# if not config.simple_output:
# html += """
# <!-- MultiQC Table Columns Modal -->
# <div class="modal fade" id="{tid}_configModal" tabindex="-1">
# <div class="modal-dialog modal-lg">
# <div class="modal-content">
# <div class="modal-header">
# <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button>
# <h4 class="modal-title">{title}: Columns</h4>
# </div>
# <div class="modal-body">
# <p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p>
# <p>
# <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button>
# <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button>
# </p>
# <table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table" data-title="{title}">
# <thead>
# <tr>
# <th class="sorthandle" style="text-align:center;">Sort</th>
# <th style="text-align:center;">Visible</th>
# <th>Group</th>
# <th>Column</th>
# <th>Description</th>
# <th>ID</th>
# <th>Scale</th>
# </tr>
# </thead>
# <tbody>
# {trows}
# </tbody>
# </table>
# </div>
# <div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div>
# </div> </div> </div>""".format( tid=table_id, title=table_title, trows=''.join(t_modal_headers.values()) )
# Save the raw values to a file if requested
if dt.pconfig.get('save_file') is True:
fn = dt.pconfig.get('raw_data_fn', 'multiqc_{}'.format(table_id))
util_functions.write_data_file(dt.raw_vals, fn)
report.saved_raw_data[fn] = dt.raw_vals
return html
def make_table(dt):
"""
Build the HTML needed for a MultiQC table.
:param data: MultiQC datatable object
"""
table_id = dt.pconfig.get(
'id', 'table_{}'.format(''.join(random.sample(letters, 4))))
table_id = report.save_htmlid(table_id)
t_headers = OrderedDict()
t_modal_headers = OrderedDict()
t_rows = OrderedDict()
dt.raw_vals = defaultdict(lambda: dict())
empty_cells = dict()
hidden_cols = 1
table_title = dt.pconfig.get('table_title')
if table_title is None:
table_title = table_id.replace("_", " ").title()
for idx, k, header in dt.get_headers_in_order():
rid = header['rid']
# Build the table header cell
shared_key = ''
if header.get('shared_key', None) is not None:
shared_key = ' data-shared-key={}'.format(header['shared_key'])
hide = ''
muted = ''
checked = ' checked="checked"'
if header.get('hidden', False) is True:
hide = 'hidden'
muted = ' text-muted'
checked = ''
hidden_cols += 1
data_attr = 'data-dmax="{}" data-dmin="{}" data-namespace="{}" {}' \
.format(header['dmax'], header['dmin'], header['namespace'], shared_key)
cell_contents = '<span class="mqc_table_tooltip" title="{}: {}">{}</span>' \
.format(header['namespace'], header['description'], header['title'])
t_headers[rid] = '<th id="header_{rid}" class="{rid} {h}" {da}>{c}</th>' \
.format(rid=rid, h=hide, da=data_attr, c=cell_contents)
empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format(
rid=rid, h=hide)
# Build the modal table row
t_modal_headers[rid] = """
<tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);">
<td class="sorthandle ui-sortable-handle">||</span></td>
<td style="text-align:center;">
<input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}">
</td>
<td>{name}</td>
<td>{title}</td>
<td>{desc}</td>
<td>{col_id}</td>
<td>{sk}</td>
</tr>""".format(rid=rid,
muted=muted,
checked=checked,
tid=table_id,
col=header['colour'],
name=header['namespace'],
title=header['title'],
desc=header['description'],
col_id='<code>{}</code>'.format(k),
sk=header.get('shared_key', ''))
# Make a colour scale
if header['scale'] == False:
c_scale = None
else:
c_scale = mqc_colour.mqc_colour_scale(header['scale'],
header['dmin'],
header['dmax'])
# Add the data table cells
for (s_name, samp) in dt.data[idx].items():
if k in samp:
val = samp[k]
kname = '{}_{}'.format(header['namespace'], rid)
dt.raw_vals[s_name][kname] = val
if 'modify' in header and callable(header['modify']):
val = header['modify'](val)
try:
dmin = header['dmin']
dmax = header['dmax']
percentage = ((float(val) - dmin) / (dmax - dmin)) * 100
percentage = min(percentage, 100)
percentage = max(percentage, 0)
except (ZeroDivisionError, ValueError):
percentage = 0
try:
valstring = str(header['format'].format(val))
except ValueError:
try:
valstring = str(header['format'].format(float(val)))
except ValueError:
valstring = str(val)
except:
valstring = str(val)
# This is horrible, but Python locale settings are worse
if config.thousandsSep_format is None:
config.thousandsSep_format = '<span class="mqc_thousandSep"></span>'
if config.decimalPoint_format is None:
config.decimalPoint_format = '.'
valstring = valstring.replace('.', 'DECIMAL').replace(
',', 'THOUSAND')
valstring = valstring.replace(
'DECIMAL', config.decimalPoint_format).replace(
'THOUSAND', config.thousandsSep_format)
# Percentage suffixes etc
valstring += header.get('suffix', '')
# Build HTML
if s_name not in t_rows:
t_rows[s_name] = dict()
t_rows[s_name][rid] = '<td class="{rid} {h}">{v}</td>'.format(
rid=rid, h=hide, v=valstring)
# else:
# if c_scale is not None:
# col = ' background-color:{};'.format(c_scale.get_colour(val))
# else:
# col = ''
# bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(percentage, col)
# val_html = '<span class="val">{}</span>'.format(valstring)
# wrapper_html = '<div class="wrapper">{}{}</div>'.format(bar_html, val_html)
#
# if s_name not in t_rows:
# t_rows[s_name] = dict()
# t_rows[s_name][rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(rid=rid, h=hide, c=wrapper_html)
# Remove header if we don't have any filled cells for it
if sum([len(rows) for rows in t_rows.values()]) == 0:
t_headers.pop(rid, None)
t_modal_headers.pop(rid, None)
logger.debug(
'Removing header {} from general stats table, as no data'.
format(k))
#
# Put everything together
#
# Buttons above the table
html = ''
if not config.simple_output:
# Copy Table Button
html += """
<button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}">
<span class="glyphicon glyphicon-copy"></span> Copy table
</button>
""".format(tid=table_id)
# Configure Columns Button
if len(t_headers) > 1:
html += """
<button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal">
<span class="glyphicon glyphicon-th"></span> Configure Columns
</button>
""".format(tid=table_id)
# Sort By Highlight button
html += """
<button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;">
<span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight
</button>
""".format(tid=table_id)
# Scatter Plot Button
if len(t_headers) > 1:
html += """
<button type="button" class="mqc_table_makeScatter btn btn-default btn-sm" data-toggle="modal" data-target="#tableScatterModal" data-table="#{tid}">
<span class="glyphicon glyphicon glyphicon-stats"></span> Plot
</button>
""".format(tid=table_id)
# "Showing x of y columns" text
html += """
<small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small>
""".format(tid=table_id,
nrows=len(t_rows),
ncols_vis=(len(t_headers) + 1) - hidden_cols,
ncols=len(t_headers))
# Build the table itself
collapse_class = 'mqc-table-collapse' if len(
t_rows) > 10 and config.collapse_tables else ''
html += """
<div id="{tid}_container" class="mqc_table_container">
<div class="table-responsive mqc-table-responsive {cc}">
<table id="{tid}" class="table table-condensed mqc_table" data-title="{title}">
""".format(tid=table_id, title=table_title, cc=collapse_class)
# Build the header row
col1_header = dt.pconfig.get('col1_header', ' ')
html += '<thead><tr><th class="rowheader">{}</th>{}</tr></thead>'.format(
col1_header, ''.join(t_headers.values()))
# Build the table body
html += '<tbody>'
t_row_keys = t_rows.keys()
if dt.pconfig.get('sortRows') is not False:
t_row_keys = sorted(t_row_keys)
for s_name in t_row_keys:
html += '<tr>'
# Sample name row header
html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format(
sn=s_name)
for k in t_headers:
html += t_rows[s_name].get(k, empty_cells[k])
html += '</tr>'
html += '</tbody></table></div>'
if len(t_rows) > 10 and config.collapse_tables:
html += '<div class="mqc-table-expand"><span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span></div>'
html += '</div>'
# Build the bootstrap modal to customise columns and order
if not config.simple_output:
html += """
<!-- MultiQC Table Columns Modal -->
<div class="modal fade" id="{tid}_configModal" tabindex="-1">
<div class="modal-dialog modal-lg">
<div class="modal-content">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button>
<h4 class="modal-title">{title}: Columns</h4>
</div>
<div class="modal-body">
<p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p>
<p>
<button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button>
<button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button>
</p>
<table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table" data-title="{title}">
<thead>
<tr>
<th class="sorthandle" style="text-align:center;">Sort</th>
<th style="text-align:center;">Visible</th>
<th>Group</th>
<th>Column</th>
<th>Description</th>
<th>ID</th>
<th>Scale</th>
</tr>
</thead>
<tbody>
{trows}
</tbody>
</table>
</div>
<div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div>
</div> </div> </div>""".format(tid=table_id,
title=table_title,
trows=''.join(t_modal_headers.values()))
# Save the raw values to a file if requested
if dt.pconfig.get('save_file') is True:
fn = dt.pconfig.get('raw_data_fn', 'multiqc_{}'.format(table_id))
util_functions.write_data_file(dt.raw_vals, fn)
report.saved_raw_data[fn] = dt.raw_vals
return html
def highcharts_heatmap(data, xcats, ycats, pconfig=None):
"""
Build the HTML needed for a HighCharts line graph. Should be
called by plot_xy_data, which properly formats input data.
"""
if pconfig is None:
pconfig = {}
# Reformat the data for highcharts
pdata = []
minval = None
maxval = None
for i, arr in enumerate(data):
for j, val in enumerate(arr):
pdata.append([j, i, val])
if minval is None or val < minval:
minval = val
if maxval is None or val > maxval:
maxval = val
if "min" not in pconfig:
pconfig["min"] = minval
if "max" not in pconfig:
pconfig["max"] = maxval
# Get the plot ID
if pconfig.get("id") is None:
pconfig["id"] = "mqc_hcplot_" + "".join(random.sample(letters, 10))
# Sanitise plot ID and check for duplicates
pconfig["id"] = report.save_htmlid(pconfig["id"])
# Build the HTML for the page
html = """
<div class="mqc_hcplot_plotgroup">
<div class="btn-group hc_switch_group">
<button type="button" class="mqc_heatmap_sortHighlight btn btn-default btn-sm" data-target="#{id}" disabled="disabled">
<span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight
</button>
</div>
<div class="mqc_hcplot_range_sliders">
<div>
<label for="{id}_range_slider_min_txt">Min:</label>
<input id="{id}_range_slider_min_txt" type="number" class="form-control" value="{min}" min="{min}" max="{max}" data-minmax="min" data-target="{id}" />
<input id="{id}_range_slider_min" type="range" value="{min}" min="{min}" max="{max}" step="any" data-minmax="min" data-target="{id}" />
</div>
<div>
<label for="{id}_range_slider_max_txt">Max:</label>
<input id="{id}_range_slider_max_txt" type="number" class="form-control" value="{max}" min="{min}" max="{max}" data-minmax="max" data-target="{id}" />
<input id="{id}_range_slider_max" type="range" value="{max}" min="{min}" max="{max}" step="any" data-minmax="max" data-target="{id}" />
</div>
</div>
<div class="hc-plot-wrapper"{height}>
<div id="{id}" class="hc-plot not_rendered hc-heatmap">
<small>loading..</small>
</div>
</div>
</div> \n""".format(
id=pconfig["id"],
min=pconfig["min"],
max=pconfig["max"],
height=f' style="height:{pconfig["height"]}px"'
if "height" in pconfig else "",
)
report.num_hc_plots += 1
report.plot_data[pconfig["id"]] = {
"plot_type": "heatmap",
"data": pdata,
"xcats": xcats,
"ycats": ycats,
"config": pconfig,
}
return html