def make_cat_etc(args, cat_root):
cat_etc = QCCategory('etc',
html_head='<h1>Other quality metrics</h1><hr>',
parent=cat_root)
cat_roadmap = QCCategory('roadmap',
html_head='<h2>Comparison to Roadmap DNase</h2>',
html_foot="""
<p>This bar chart shows the correlation between the Roadmap DNase samples to
your sample, when the signal in the universal DNase peak region sets are
compared. The closer the sample is in signal distribution in the regions
to your sample, the higher the correlation.</p><br>
""",
parent=cat_etc)
if args.roadmap_compare_plots:
for i, plot in enumerate(args.roadmap_compare_plots):
if plot:
cat_roadmap.add_plot(plot, key=str_rep(i), size_pct=50)
return cat_etc
def make_cat_peak_stat(args, cat_root):
cat_peak_stat = QCCategory(
'peak_stat',
html_head='<h1>Peak calling statistics</h1><hr>',
parent=cat_root
)
cat_peak_region_size = QCCategory(
'peak_region_size',
html_head='<h2>Peak region size</h2>',
html_foot="""
""",
parser=parse_peak_region_size_qc,
map_key_desc=MAP_KEY_DESC_PEAK_REGION_SIZE_QC,
parent=cat_peak_stat,
)
if args.peak_region_size_qcs:
for i, qc in enumerate(args.peak_region_size_qcs):
if qc:
cat_peak_region_size.add_log(qc, key=str_rep(i))
if args.idr_opt_peak_region_size_qc:
qc = args.idr_opt_peak_region_size_qc[0]
cat_peak_region_size.add_log(qc, key='idr_opt')
if args.overlap_opt_peak_region_size_qc:
qc = args.overlap_opt_peak_region_size_qc[0]
cat_peak_region_size.add_log(qc, key='overlap_opt')
# plots
if args.peak_region_size_plots:
for i, plot in enumerate(args.peak_region_size_plots):
if plot:
cat_peak_region_size.add_plot(plot, key=str_rep(i),
size_pct=35)
if args.idr_opt_peak_region_size_plot:
plot = args.idr_opt_peak_region_size_plot[0]
cat_peak_region_size.add_plot(plot, key='idr_opt', size_pct=35)
if args.overlap_opt_peak_region_size_plot:
plot = args.overlap_opt_peak_region_size_plot[0]
cat_peak_region_size.add_plot(plot, key='overlap_opt', size_pct=35)
return cat_peak_stat
def make_cat_align_enrich(args, cat_root):
cat_align_enrich = QCCategory(
'align_enrich',
html_head='<h1>Enrichment / Signal-to-noise ratio</h1><hr>',
parent=cat_root
)
if args.pipeline_type in ('tf', 'histone'):
html_head_xcor = '<h2>Strand cross-correlation measures (trimmed/filtered SE BAM)</h2>'
html_foot_xcor = """
<br><p>Performed on subsampled ({xcor_subsample_reads}) reads mapped from FASTQs that are trimmed to {xcor_pe_trim_bp}.
Such FASTQ trimming and subsampling reads are for cross-corrleation analysis only.
Untrimmed FASTQs are used for all the other analyses.</p>
<div id='help-xcor'><p>
NOTE1: For SE datasets, reads from replicates are randomly subsampled to {xcor_subsample_reads}.<br>
NOTE2: For PE datasets, the first end (R1) of each read-pair is selected and trimmed to {xcor_pe_trim_bp} the reads are then randomly subsampled to {xcor_subsample_reads}.<br>
""".format(
xcor_subsample_reads=args.xcor_subsample_reads,
xcor_pe_trim_bp=args.xcor_pe_trim_bp,
)
else:
html_head_xcor = '<h2>Strand cross-correlation measures (filtered BAM)</h2>'
html_foot_xcor = """
<br><p>Performed on subsampled ({xcor_subsample_reads}) reads.
Such FASTQ trimming is for cross-corrleation analysis only.</p>
<div id='help-xcor'><p>
""".format(
xcor_subsample_reads=args.xcor_subsample_reads
)
html_foot_xcor += """<ul>
<li>Normalized strand cross-correlation coefficient (NSC) = col9 in outFile </li>
<li>Relative strand cross-correlation coefficient (RSC) = col10 in outFile </li>
<li>Estimated fragment length = col3 in outFile, take the top value </li>
</ul></p></div><br>
"""
cat_xcor = QCCategory(
'xcor_score',
html_head=html_head_xcor,
html_foot=html_foot_xcor,
parser=parse_xcor_score,
map_key_desc=MAP_KEY_DESC_XCOR_SCORE,
parent=cat_align_enrich,
)
if args.xcor_scores:
for i, qc in enumerate(args.xcor_scores):
if qc:
cat_xcor.add_log(qc, key=str_rep(i))
# trivial subcategory to show table legend before plots
cat_xcor_plot = QCCategory(
'xcor_plot',
parent=cat_align_enrich,
)
if args.xcor_plots:
for i, plot in enumerate(args.xcor_plots):
if plot:
cat_xcor_plot.add_plot(plot, key=str_rep(i), size_pct=60)
cat_tss_enrich = QCCategory(
'tss_enrich',
html_head='<h2>TSS enrichment (filtered/deduped BAM)</h2>',
html_foot="""
<p>Open chromatin assays should show enrichment in open chromatin sites, such as
TSS's. An average TSS enrichment in human (hg19) is above 6. A strong TSS enrichment is
above 10. For other references please see https://www.encodeproject.org/atac-seq/</p><br>
""",
parser=parse_tss_enrich_qc,
map_key_desc=MAP_KEY_DESC_TSS_ENRICH_QC,
parent=cat_align_enrich
)
if args.tss_enrich_qcs:
for i, qc in enumerate(args.tss_enrich_qcs):
if qc:
cat_tss_enrich.add_log(qc, key=str_rep(i))
if args.tss_large_plots:
for i, plot in enumerate(args.tss_large_plots):
if plot:
cat_tss_enrich.add_plot(plot, key=str_rep(i))
cat_jsd = QCCategory(
'jsd',
html_head='<h2>Jensen-Shannon distance (filtered/deduped BAM)</h2>',
parser=parse_jsd_qc,
map_key_desc=MAP_KEY_DESC_JSD_QC,
parent=cat_align_enrich
)
if args.jsd_plot:
plot = args.jsd_plot[0]
cat_jsd.add_plot(plot, size_pct=40)
if args.jsd_qcs:
for i, qc in enumerate(args.jsd_qcs):
if qc:
cat_jsd.add_log(qc, key=str_rep(i))
return cat_align_enrich
def make_cat_replication(args, cat_root):
cat_replication = QCCategory(
'replication',
html_head='<h1>Replication quality metrics</h1><hr>',
parent=cat_root
)
cat_idr = QCCategory(
'idr',
html_head='<h2>IDR (Irreproducible Discovery Rate) plots</h2>',
parent=cat_replication,
)
if args.idr_plots:
num_rep = infer_n_from_nC2(len(args.idr_plots))
for i, plot in enumerate(args.idr_plots):
if plot:
cat_idr.add_plot(
plot,
key=infer_pair_label_from_idx(num_rep, i))
if args.idr_plots_pr:
for i, plot in enumerate(args.idr_plots_pr):
if plot:
cat_idr.add_plot(
plot,
key='rep{X}-pr1_vs_rep{X}-pr2'.format(X=i+1))
if args.idr_plot_ppr:
cat_idr.add_plot(args.idr_plot_ppr[0], key='pooled-pr1_vs_pooled-pr2')
cat_reproducibility = QCCategory(
'reproducibility',
html_head='<h2>Reproducibility QC and peak detection statistics</h2>',
html_foot="""
<div id='help-reproducibility'><p>Reproducibility QC<br>
<ul>
<li>N1: Replicate 1 self-consistent peaks (comparing two pseudoreplicates generated by subsampling Rep1 reads) </li>
<li>N2: Replicate 2 self-consistent peaks (comparing two pseudoreplicates generated by subsampling Rep2 reads) </li>
<li>Ni: Replicate i self-consistent peaks (comparing two pseudoreplicates generated by subsampling RepX reads) </li>
<li>Nt: True Replicate consistent peaks (comparing true replicates Rep1 vs Rep2) </li>
<li>Np: Pooled-pseudoreplicate consistent peaks (comparing two pseudoreplicates generated by subsampling pooled reads from Rep1 and Rep2) </li>
<li>Self-consistency Ratio: max(N1,N2) / min (N1,N2) </li>
<li>Rescue Ratio: max(Np,Nt) / min (Np,Nt) </li>
<li>Reproducibility Test: If Self-consistency Ratio >2 AND Rescue Ratio > 2, then 'Fail' else 'Pass' </li>
</ul></p></div><br>
""",
parser=parse_reproducibility_qc,
map_key_desc=MAP_KEY_DESC_REPRODUCIBILITY_QC,
parent=cat_replication,
)
if args.overlap_reproducibility_qc:
qc = args.overlap_reproducibility_qc[0]
cat_reproducibility.add_log(qc, key='overlap')
if args.idr_reproducibility_qc:
qc = args.idr_reproducibility_qc[0]
cat_reproducibility.add_log(qc, key='idr')
if args.peak_caller == 'spp':
extra_info = 'with FDR 0.01'
elif args.peak_caller == 'macs2':
extra_info = 'with p-val threshold {}'.format(args.pval_thresh)
else:
extra_info = ''
cat_num_peak = QCCategory(
'num_peaks',
html_head='<h2>Number of raw peaks</h2>',
html_foot="""
Top {num_peak} raw peaks from {peak_caller} {extra_info}
""".format(
num_peak=args.cap_num_peak,
peak_caller=args.peak_caller,
extra_info=extra_info,
),
parser=parse_num_peak_qc,
map_key_desc=MAP_KEY_DESC_NUM_PEAK_QC,
parent=cat_replication,
)
if args.num_peak_qcs:
for i, qc in enumerate(args.num_peak_qcs):
if qc:
cat_num_peak.add_log(qc, key=str_rep(i))
return cat_replication
def make_cat_align(args, cat_root):
cat_align = QCCategory(
'align',
html_head='<h1>Alignment quality metrics</h1><hr>',
parent=cat_root
)
cat_align_samstat = QCCategory(
'samstat',
html_head='<h2>SAMstat (raw unfiltered BAM)</h2>',
parser=parse_flagstat_qc,
map_key_desc=MAP_KEY_DESC_FLAGSTAT_QC,
parent=cat_align
)
if args.samstat_qcs:
for i, qc in enumerate(args.samstat_qcs):
if qc:
cat_align_samstat.add_log(qc, key=str_rep(i))
if args.ctl_samstat_qcs:
for i, qc in enumerate(args.ctl_samstat_qcs):
if qc:
cat_align_samstat.add_log(qc, key=str_ctl(i))
cat_align_dup = QCCategory(
'dup',
html_head='<h2>Marking duplicates (filtered BAM)</h2>',
html_foot="""
<div id='help-filter'>
Filtered out (samtools view -F 1804):
<ul>
<li>read unmapped (0x4)</li>
<li>mate unmapped (0x8, for paired-end)</li>
<li>not primary alignment (0x100)</li>
<li>read fails platform/vendor quality checks (0x200)</li>
<li>read is PCR or optical duplicate (0x400)</li>
</ul></p></div><br>
""",
parser=parse_dup_qc,
map_key_desc=MAP_KEY_DESC_DUP_QC,
parent=cat_align
)
if args.dup_qcs:
for i, qc in enumerate(args.dup_qcs):
if qc:
cat_align_dup.add_log(qc, key=str_rep(i))
if args.ctl_dup_qcs:
for i, qc in enumerate(args.ctl_dup_qcs):
if qc:
cat_align_dup.add_log(qc, key=str_ctl(i))
cat_align_frac_mito = QCCategory(
'frac_mito',
html_head='<h2>Fraction of mitochondrial reads (unfiltered BAM)</h2>',
parser=parse_frac_mito_qc,
map_key_desc=MAP_KEY_DESC_FRAC_MITO_QC,
parent=cat_align
)
if args.frac_mito_qcs:
for i, qc in enumerate(args.frac_mito_qcs):
if qc:
cat_align_frac_mito.add_log(qc, key=str_rep(i))
if args.ctl_frac_mito_qcs:
for i, qc in enumerate(args.ctl_frac_mito_qcs):
if qc:
cat_align_frac_mito.add_log(qc, key=str_ctl(i))
cat_align_preseq = QCCategory(
'preseq',
html_foot="""
<p>Preseq performs a yield prediction by subsampling the reads, calculating the
number of distinct reads, and then extrapolating out to see where the
expected number of distinct reads no longer increases. The confidence interval
gives a gauge as to the validity of the yield predictions.</p>
""",
parent=cat_align
)
if args.preseq_plots:
for i, plot in enumerate(args.preseq_plots):
if plot:
cat_align_preseq.add_plot(
plot, key=str_rep(i), size_pct=50)
cat_align_nodup_samstat = QCCategory(
'nodup_samstat',
html_head='<h2>SAMstat (filtered/deduped BAM)</h2>',
html_foot="""
<p>Filtered and duplicates removed</p><br>
""",
parser=parse_flagstat_qc,
map_key_desc=MAP_KEY_DESC_FLAGSTAT_QC,
parent=cat_align
)
if args.nodup_samstat_qcs:
for i, qc in enumerate(args.nodup_samstat_qcs):
if qc:
cat_align_nodup_samstat.add_log(qc, key=str_rep(i))
if args.ctl_nodup_samstat_qcs:
for i, qc in enumerate(args.ctl_nodup_samstat_qcs):
if qc:
cat_align_nodup_samstat.add_log(qc, key=str_ctl(i))
cat_fraglen = QCCategory(
'frag_len_stat',
html_head='<h2>Fragment length statistics (filtered/deduped BAM)</h2>',
html_foot="""
<p>Open chromatin assays show distinct fragment length enrichments, as the cut
sites are only in open chromatin and not in nucleosomes. As such, peaks
representing different n-nucleosomal (ex mono-nucleosomal, di-nucleosomal)
fragment lengths will arise. Good libraries will show these peaks in a
fragment length distribution and will show specific peak ratios.</p><br>
<ul>
<li>NFR: Nucleosome free region
</ul><br>
""",
parser=parse_nucleosomal_qc,
map_key_desc=MAP_KEY_DESC_NUCLEOSOMAL_QC,
parent=cat_align
)
if args.fraglen_nucleosomal_qcs:
for i, qc in enumerate(args.fraglen_nucleosomal_qcs):
if qc:
cat_fraglen.add_log(qc, key=str_rep(i))
if args.fraglen_dist_plots:
for i, plot in enumerate(args.fraglen_dist_plots):
if plot:
cat_fraglen.add_plot(plot,
key=str_rep(i), size_pct=50)
cat_gc_bias = QCCategory(
'gc_bias',
html_head='<h2>Sequence quality metrics (filtered/deduped BAM)</h2>',
html_foot="""
<p>Open chromatin assays are known to have significant GC bias. Please take this
into consideration as necessary.</p><br>
""",
parent=cat_align
)
if args.gc_plots:
for i, plot in enumerate(args.gc_plots):
if plot:
cat_gc_bias.add_plot(plot,
key=str_rep(i), size_pct=60)
return cat_align
def make_cat_align_enrich(args, cat_root):
cat_align_enrich = QCCategory(
'align_enrich',
html_head='<h1>Enrichment / Signal-to-noise ratio</h1><hr>',
parent=cat_root
)
cat_xcor = QCCategory(
'xcor_score',
html_head='<h2>Strand cross-correlation measures</h2>',
html_foot="""
<br><p>Performed on subsampled reads</p>
<div id='help-xcor'><p>
NOTE1: For SE datasets, reads from replicates are randomly subsampled.<br>
NOTE2: For PE datasets, the first end of each read-pair is selected and the reads are then randomly subsampled.<br>
<ul>
<li>Normalized strand cross-correlation coefficient (NSC) = col9 in outFile </li>
<li>Relative strand cross-correlation coefficient (RSC) = col10 in outFile </li>
<li>Estimated fragment length = col3 in outFile, take the top value </li>
</ul></p></div><br>
""",
parser=parse_xcor_score,
map_key_desc=MAP_KEY_DESC_XCOR_SCORE,
parent=cat_align_enrich,
)
if args.xcor_scores:
for i, qc in enumerate(args.xcor_scores):
if qc:
cat_xcor.add_log(qc, key=str_rep(i))
# trivial subcategory to show table legend before plots
cat_xcor_plot = QCCategory(
'xcor_plot',
parent=cat_align_enrich,
)
if args.xcor_plots:
for i, plot in enumerate(args.xcor_plots):
if plot:
cat_xcor_plot.add_plot(plot, key=str_rep(i), size_pct=60)
cat_tss_enrich = QCCategory(
'tss_enrich',
html_head='<h2>TSS enrichment</h2>',
html_foot="""
<p>Open chromatin assays should show enrichment in open chromatin sites, such as
TSS's. An average TSS enrichment in human (hg19) is above 6. A strong TSS enrichment is
above 10. For other references please see https://www.encodeproject.org/atac-seq/</p><br>
""",
parser=parse_tss_enrich_qc,
map_key_desc=MAP_KEY_DESC_TSS_ENRICH_QC,
parent=cat_align_enrich
)
if args.tss_enrich_qcs:
for i, qc in enumerate(args.tss_enrich_qcs):
if qc:
cat_tss_enrich.add_log(qc, key=str_rep(i))
if args.tss_large_plots:
for i, plot in enumerate(args.tss_large_plots):
if plot:
cat_tss_enrich.add_plot(plot, key=str_rep(i))
cat_jsd = QCCategory(
'jsd',
html_head='<h2>Jensen-Shannon distance</h2>',
parser=parse_jsd_qc,
map_key_desc=MAP_KEY_DESC_JSD_QC,
parent=cat_align_enrich
)
if args.jsd_plot:
plot = args.jsd_plot[0]
cat_jsd.add_plot(plot, size_pct=40)
if args.jsd_qcs:
for i, qc in enumerate(args.jsd_qcs):
if qc:
cat_jsd.add_log(qc, key=str_rep(i))
return cat_align_enrich
def make_cat_align(args, cat_root):
cat_align = QCCategory('align',
html_head='<h1>Alignment quality metrics</h1><hr>',
parent=cat_root)
cat_align_samstat = QCCategory(
'samstat',
html_head='<h2>SAMstat (raw unfiltered BAM)</h2>',
parser=parse_flagstat_qc,
map_key_desc=MAP_KEY_DESC_FLAGSTAT_QC,
parent=cat_align)
if args.samstat_qcs:
for i, qc in enumerate(args.samstat_qcs):
if qc:
cat_align_samstat.add_log(qc, key=str_rep(i))
if args.ctl_samstat_qcs:
for i, qc in enumerate(args.ctl_samstat_qcs):
if qc:
cat_align_samstat.add_log(qc, key=str_ctl(i))
cat_align_dup = QCCategory(
'dup',
html_head='<h2>Marking duplicates (filtered BAM)</h2>',
html_foot="""
<div id='help-filter'>
Filtered with samtools flag 1804 (samtools view -F 1804):
<ul>
<li>read unmapped (0x4)</li>
<li>mate unmapped (0x8, for paired-end)</li>
<li>not primary alignment (0x100)</li>
<li>read fails platform/vendor quality checks (0x200)</li>
<li>read is PCR or optical duplicate (0x400)</li>
</ul></p></div><br>
""",
parser=parse_dup_qc,
map_key_desc=MAP_KEY_DESC_DUP_QC,
parent=cat_align)
if args.dup_qcs:
for i, qc in enumerate(args.dup_qcs):
if qc:
cat_align_dup.add_log(qc, key=str_rep(i))
if args.ctl_dup_qcs:
for i, qc in enumerate(args.ctl_dup_qcs):
if qc:
cat_align_dup.add_log(qc, key=str_ctl(i))
cat_align_frac_mito = QCCategory(
'frac_mito',
html_head='<h2>Fraction of mitochondrial reads (unfiltered BAM)</h2>',
parser=parse_frac_mito_qc,
map_key_desc=MAP_KEY_DESC_FRAC_MITO_QC,
parent=cat_align)
if args.frac_mito_qcs:
for i, qc in enumerate(args.frac_mito_qcs):
if qc:
cat_align_frac_mito.add_log(qc, key=str_rep(i))
if args.ctl_frac_mito_qcs:
for i, qc in enumerate(args.ctl_frac_mito_qcs):
if qc:
cat_align_frac_mito.add_log(qc, key=str_ctl(i))
cat_align_preseq = QCCategory('preseq',
html_foot="""
<p>Preseq performs a yield prediction by subsampling the reads, calculating the
number of distinct reads, and then extrapolating out to see where the
expected number of distinct reads no longer increases. The confidence interval
gives a gauge as to the validity of the yield predictions.</p>
""",
parent=cat_align)
if args.preseq_plots:
for i, plot in enumerate(args.preseq_plots):
if plot:
cat_align_preseq.add_plot(plot, key=str_rep(i), size_pct=50)
cat_align_nodup_samstat = QCCategory(
'nodup_samstat',
html_head='<h2>SAMstat (filtered/deduped BAM)</h2>',
html_foot="""
<p>Filtered {dup_removal_detail}.
Subsampling with {pipeline_prefix}.{subsample_param_name} is not done in alignment steps.
Nodup BAM is converted into a BED type (TAGALIGN) later and then TAGALIGN is subsampled
with such parameter in the peak-calling step.<br>
</p>
""".format(
dup_removal_detail='but duplicates are kept'
if args.no_dup_removal else 'and duplicates are removed',
pipeline_prefix=args.pipeline_prefix,
subsample_param_name='subsample_reads',
),
parser=parse_flagstat_qc,
map_key_desc=MAP_KEY_DESC_FLAGSTAT_QC,
parent=cat_align)
if args.nodup_samstat_qcs:
for i, qc in enumerate(args.nodup_samstat_qcs):
if qc:
cat_align_nodup_samstat.add_log(qc, key=str_rep(i))
if args.ctl_nodup_samstat_qcs:
for i, qc in enumerate(args.ctl_nodup_samstat_qcs):
if qc:
cat_align_nodup_samstat.add_log(qc, key=str_ctl(i))
cat_fraglen = QCCategory(
'frag_len_stat',
html_head='<h2>Fragment length statistics (filtered/deduped BAM)</h2>',
html_foot="""
<p>Open chromatin assays show distinct fragment length enrichments, as the cut
sites are only in open chromatin and not in nucleosomes. As such, peaks
representing different n-nucleosomal (ex mono-nucleosomal, di-nucleosomal)
fragment lengths will arise. Good libraries will show these peaks in a
fragment length distribution and will show specific peak ratios.</p><br>
<ul>
<li>NFR: Nucleosome free region
</ul><br>
""",
parser=parse_nucleosomal_qc,
map_key_desc=MAP_KEY_DESC_NUCLEOSOMAL_QC,
parent=cat_align)
if args.fraglen_nucleosomal_qcs:
for i, qc in enumerate(args.fraglen_nucleosomal_qcs):
if qc:
cat_fraglen.add_log(qc, key=str_rep(i))
if args.fraglen_dist_plots:
for i, plot in enumerate(args.fraglen_dist_plots):
if plot:
cat_fraglen.add_plot(plot, key=str_rep(i), size_pct=50)
cat_gc_bias = QCCategory(
'gc_bias',
html_head='<h2>Sequence quality metrics (filtered/deduped BAM)</h2>',
html_foot="""
<p>Open chromatin assays are known to have significant GC bias. Please take this
into consideration as necessary.</p><br>
""",
parent=cat_align)
if args.gc_plots:
for i, plot in enumerate(args.gc_plots):
if plot:
cat_gc_bias.add_plot(plot, key=str_rep(i), size_pct=60)
cat_annot_enrich = QCCategory(
'frac_reads_in_annot',
html_head='<h2>Annotated genomic region enrichment</h2>',
html_foot="""
<p>Signal to noise can be assessed by considering whether reads are falling into
known open regions (such as DHS regions) or not. A high fraction of reads
should fall into the universal (across cell type) DHS set. A small fraction
should fall into the blacklist regions. A high set (though not all) should
fall into the promoter regions. A high set (though not all) should fall into
the enhancer regions. The promoter regions should not take up all reads, as
it is known that there is a bias for promoters in open chromatin assays.</p><br>
""",
parser=parse_annot_enrich_qc,
map_key_desc=MAP_KEY_DESC_ANNOT_ENRICH_QC,
parent=cat_align,
)
if args.annot_enrich_qcs:
for i, qc in enumerate(args.annot_enrich_qcs):
if qc:
cat_annot_enrich.add_log(qc, key=str_rep(i))
return cat_align
