def test_large_dataset(self):
SUBREADS = "/pbi/dept/secondary/siv/testdata/SA3-Sequel/phi29/315/3150101/r54008_20160219_002905/1_A01_tiny_barcoded/m54008_160219_003234.tiny.subreadset.xml"
BARCODES = "/pbi/dept/secondary/siv/barcodes/Sequel_RSII_384_barcodes_v1/Sequel_RSII_384_barcodes_v1.barcodeset.xml"
SUBREADS_IN = "/pbi/dept/secondary/siv/testdata/SA3-Sequel/phi29/315/3150101/r54008_20160219_002905/1_A01_tiny/m54008_160219_003234_tiny.subreadset.xml"
report = run_to_report(SUBREADS, BARCODES, SUBREADS_IN)
validate_report_complete(self, report)
d = report.to_dict()
self.assertIsNotNone(d)
attr = {a.id: a.value for a in report.attributes}
self.assertEqual(attr, {
'mean_read_length': 14411,
'mean_longest_subread_length': 28314,
'min_reads': 989,
'mean_reads': 1065,
'n_barcodes': 3,
'max_reads': 1116,
'n_barcoded_reads': 3196,
'n_unbarcoded_reads': 0
})
self.assertEqual(report.tables[0].columns[1].values,
["0--0", "1--1", "2--2"])
self.assertEqual(report.tables[0].columns[2].values,
['bc1001--bc1001', 'bc1002--bc1002', 'bc1003--bc1003'])
self.assertEqual(report.tables[0].columns[3].values, [1091, 1116, 989])
self.assertEqual(report.tables[0].columns[
4].values, [5370, 5053, 4710])
self.assertEqual(report.tables[0].columns[5].values, [
10306688, 10034254, 9452616])
def test_make_modifications_report(self):
"""
Create a report object, do assertions on its properties, then assert that it gets written
"""
h5 = os.path.join(self._data_dir, "basemods.h5")
make_modifications_report(h5, 'rpt.json', self._output_dir, 60)
# deserialize report
log.info("Deserializing report {r}".format(
r=os.path.join(self._output_dir, 'rpt.json')))
s = None
with open(os.path.join(self._output_dir, 'rpt.json'), 'r') as f:
s = json.load(f)
report = dict_to_report(s)
validate_report_complete(self, report)
self.assertEqual(1, len(report.plotGroups))
self.assertEqual(2, len(report.plotGroups[0].plots))
scatter = report.plotGroups[0].plots[0]
histo = report.plotGroups[0].plots[1]
self._assert_image_file(report.plotGroups[0].thumbnail)
self._assert_image_file(scatter.image)
self._assert_image_file(histo.image)
def test_basic(self):
"""Basic smoke test"""
output_dir = tempfile.mkdtemp()
report = to_report(self.ccs_set, output_dir)
validate_report_complete(self, report)
d = report.to_dict()
self.assertIsNotNone(d)
def test_make_filter_stats_report_sts_xml(self):
"""
Test the content of the filter report generated from an sts.xml
"""
sts_xml = pbtestdata.get_file("stats-xml")
rpt = make_filter_report(sts_xml, self.get_output_dir())
d = json.loads(rpt.to_json())
self._compare_attribute_values(report_d=d,
expected_d={
Constants.A_NBASES: 1672335649,
Constants.A_NREADS: 394658,
Constants.A_READ_N50: 7750,
Constants.A_READ_LENGTH: 4237
})
self.assertTrue(
os.path.exists(
os.path.join(self.get_output_dir(), 'readLenDist0.png')))
# self.assertTrue(os.path.exists(os.path.join(
# self.get_output_dir(),
# 'readQualDist0.png')))
# these are from a raw STS file
self.assertEqual(len(rpt._dataset_uuids), 0,
"Incorrect report datasets uuids")
print pformat(rpt.to_dict())
validate_report_complete(self, rpt)
def test_make_modifications_report(self):
"""
Create a report object, do assertions on its properties, then assert that it gets written
"""
h5 = os.path.join(self._data_dir, "basemods.h5")
make_modifications_report(h5, 'rpt.json', self._output_dir, 60)
# deserialize report
log.info("Deserializing report {r}".format(
r=os.path.join(self._output_dir, 'rpt.json')))
s = None
with open(os.path.join(self._output_dir, 'rpt.json'), 'r') as f:
s = json.load(f)
report = dict_to_report(s)
validate_report_complete(self, report)
self.assertEqual(1, len(report.plotGroups))
self.assertEqual(2, len(report.plotGroups[0].plots))
scatter = report.plotGroups[0].plots[0]
histo = report.plotGroups[0].plots[1]
self._assert_image_file(report.plotGroups[0].thumbnail)
self._assert_image_file(scatter.image)
self._assert_image_file(histo.image)
def test_exit_code_0(self):
bam = self.getAlignmentSet()
var_rpt = os.path.join(DATA, 'variants_report.json')
mapping_rpt = os.path.join(DATA, 'mapping_stats_report.json')
cmd = 'python -m pbreports.report.sat {o} {r} {c} {a} {v}'.format(
o=self._output_dir, r='rpt.json', c=bam, a=var_rpt, v=mapping_rpt)
o, c, m = backticks(cmd)
log.info(cmd)
if c is not 0:
log.error(m)
log.error(o)
print(m)
self.assertEquals(0, c)
rpt_file = os.path.join(self._output_dir, 'rpt.json')
rpt = load_report_from_json(rpt_file)
self.assertEqual('sidney', rpt.get_attribute_by_id('instrument').value)
self.assertEqual(1, rpt.get_attribute_by_id('coverage').value)
self.assertEqual(1, rpt.get_attribute_by_id('concordance').value)
self.assertEqual(
7752,
rpt.get_attribute_by_id('mapped_readlength_mean').value)
self.assertEqual(48, rpt.get_attribute_by_id('reads_in_cell').value)
out = StringIO()
self.assertTrue(summarize_report(rpt_file, out=out))
validate_report_complete(self, rpt)
def test_make_filter_stats_report_sts_xml(self):
"""
Test the content of the filter report generated from an sts.xml
"""
sts_xml = pbtestdata.get_file("stats-xml")
rpt = make_filter_report(sts_xml, self.get_output_dir())
d = json.loads(rpt.to_json())
self._compare_attribute_values(
report_d=d,
expected_d={
Constants.A_NBASES: 1672335649,
Constants.A_NREADS: 394658,
Constants.A_READ_N50: 7750,
Constants.A_READ_LENGTH: 4237,
},
)
self.assertTrue(os.path.exists(os.path.join(self.get_output_dir(), "readLenDist0.png")))
# self.assertTrue(os.path.exists(os.path.join(
# self.get_output_dir(),
# 'readQualDist0.png')))
# these are from a raw STS file
self.assertEqual(len(rpt._dataset_uuids), 0, "Incorrect report datasets uuids")
print pformat(rpt.to_dict())
validate_report_complete(self, rpt)
def test_basic(self):
"""Basic smoke test"""
output_dir = tempfile.mkdtemp()
report = to_report(self.ccs_set, output_dir)
validate_report_complete(self, report)
d = report.to_dict()
self.assertIsNotNone(d)
def test_make_control_report(self):
"""
Test that the report gets written proppa
"""
cmph5 = os.path.join(self.DATA_DIR, 'control_reads.cmp.h5')
csv = os.path.join(self.DATA_DIR, 'filtered_summary.csv')
make_control_report(cmph5, csv, 'foo.json',
self._output_dir, 60, False)
# deserialize report
s = None
with open(os.path.join(self._output_dir, 'foo.json'), 'r') as f:
s = json.load(f)
report = dict_to_report(s)
validate_report_complete(self, report)
self.assertEqual(7, len(report.attributes))
for i in range(2):
pg = report.plotGroups[i]
self.assertFalse(os.path.isabs(pg.thumbnail))
self.assertTrue(os.path.exists(
os.path.join(self._output_dir, pg.thumbnail)))
p = pg.plots[0]
self.assertFalse(os.path.isabs(p.image))
self.assertTrue(os.path.exists(
os.path.join(self._output_dir, p.image)))
def test_smoke(self):
"""Basic functionality test"""
self.assertIsNotNone(self.report)
d = self.report.to_dict()
self.assertTrue(isinstance(d, dict))
self.assertTrue('attributes' in d)
validate_report_complete(self, self.report)
def test_basic(self):
report = run_to_report(self.subreads, self.barcodes, subreads=True)
validate_report_complete(self, report)
d = report.to_dict()
self.assertIsNotNone(d)
self.assertEqual(report.tables[0].columns[0].values, ['0--0', '2--2'])
self.assertEqual(report.tables[0].columns[1].values, [1, 1])
self.assertEqual(report.tables[0].columns[2].values, [1436, 204])
def test_make_coverage_report(self):
"""
Create a report object, do assertions on its properties, then assert that it gets written
"""
report = make_coverage_report(self.GFF, self.REFERENCE, 25, 'rpt.json',
self._output_dir)
self.assertEqual(2, len(report.attributes))
self.assertEqual(0, len(report.tables))
self.assertEqual(2, len(report.plotGroups))
self.assertTrue(op.exists(op.join(self._output_dir, 'rpt.json')))
validate_report_complete(self, report)
def test_make_coverage_report(self):
"""
Create a report object, do assertions on its properties, then assert that it gets written
"""
report = make_coverage_report(self.GFF, self.REFERENCE, 25, 'rpt.json',
self._output_dir)
self.assertEqual(2, len(report.attributes))
self.assertEqual(0, len(report.tables))
self.assertEqual(2, len(report.plotGroups))
self.assertTrue(op.exists(op.join(self._output_dir, 'rpt.json')))
validate_report_complete(self, report)
def test_make_report(self):
report = run_to_report(self.csv_file)
validate_report_complete(self, report)
self.assertTrue(isinstance(report, Report))
log.info(pformat(report.to_dict()))
self.assertIsNotNone(report)
for j in range(len(report.tables[0].columns[0].values)):
total = 0
for col in report.tables[0].columns:
if col.id.endswith("_pct"):
total += col.values[j]
self.assertAlmostEqual(total, 1.0, places=3)
def test_make_variants_report(self):
"""
Create a report object, do assertions on its properties, then assert that it gets written
"""
report = make_variants_report(self.ALIGNMENT_SUMMARY, self.VARIANTS_GFF,
self.REFERENCE, 25, 'rpt.json', self._output_dir, 60, False)
self.assertEqual(5, len(report.attributes))
self.assertEqual(1, len(report.tables))
self.assertEqual(1, len(report.plotGroups))
self.assertIsNotNone(report.plotGroups[0].legend)
self.assertTrue(op.exists(op.join(self._output_dir, 'rpt.json')))
validate_report_complete(self, report)
def test_make_variants_report(self):
"""
Create a report object, do assertions on its properties, then assert that it gets written
"""
report = make_variants_report(self.ALIGNMENT_SUMMARY, self.VARIANTS_GFF,
self.REFERENCE, 25, 'rpt.json', self._output_dir, 60, False)
self.assertEqual(5, len(report.attributes))
self.assertEqual(1, len(report.tables))
self.assertEqual(1, len(report.plotGroups))
self.assertIsNotNone(report.plotGroups[0].legend)
self.assertTrue(op.exists(op.join(self._output_dir, 'rpt.json')))
validate_report_complete(self, report)
def test_make_report(self):
report = run_to_report(self.csv_file)
validate_report_complete(self, report)
self.assertTrue(isinstance(report, Report))
log.info(pformat(report.to_dict()))
self.assertIsNotNone(report)
for j in range(len(report.tables[0].columns[0].values)):
total = 0
for col in report.tables[0].columns:
if col.id.endswith("_pct"):
total += col.values[j]
self.assertAlmostEqual(total, 1.0, places=3)
def test_make_mv_sample_table(self):
juliet_summary = op.join(_DATA_DIR, 'mix.json')
csv_file = tempfile.NamedTemporaryFile(suffix=".csv").name
rpt = to_report(juliet_summary, csv_file, self._output_dir)
d = json.loads(rpt.to_json())
t = d['tables'][0]
c0 = t['columns'][0]
c1 = t['columns'][1]
c2 = t['columns'][2]
c3 = t['columns'][3]
c4 = t['columns'][4]
c5 = t['columns'][5]
c6 = t['columns'][6]
self.assertEqual('Barcode Name', c0['header'])
self.assertEqual('minor_variants.sample_table.barcode', c0['id'])
self.assertEqual('1--1', c0['values'][0])
self.assertEqual('0--0', c0['values'][1])
self.assertEqual('Median Coverage', c1['header'])
self.assertEqual('minor_variants.sample_table.coverage', c1['id'])
self.assertEqual(2911, c1['values'][0])
self.assertEqual(2877, c1['values'][1])
self.assertEqual('Number of Variants', c2['header'])
self.assertEqual('minor_variants.sample_table.variants', c2['id'])
self.assertEqual(4, c2['values'][0])
self.assertEqual(5, c2['values'][1])
self.assertEqual('Number of Genes', c3['header'])
self.assertEqual('minor_variants.sample_table.genes', c3['id'])
self.assertEqual(1, c3['values'][0])
self.assertEqual(1, c3['values'][1])
self.assertEqual('Number of Affected Drugs', c4['header'])
self.assertEqual('minor_variants.sample_table.drms', c4['id'])
self.assertEqual(7, c4['values'][0])
self.assertEqual(4, c4['values'][1])
self.assertEqual('Number of Haplotypes', c5['header'])
self.assertEqual('minor_variants.sample_table.haplotypes', c5['id'])
self.assertEqual(4, c5['values'][0])
self.assertEqual(5, c5['values'][1])
self.assertEqual('Maximum Frequency Haplotype (%)', c6['header'])
self.assertEqual('minor_variants.sample_table.haplotype_frequency',
c6['id'])
self.assertAlmostEqual(4.25456790123457, c6['values'][0], delta=.0003)
self.assertAlmostEqual(95.8553791887125, c6['values'][1], delta=.0003)
validate_report_complete(self, rpt)
def test_lima_output(self):
SUBREADS = "/pbi/dept/secondary/siv/testdata/pbreports-unittest/data/barcode/lima/file.datastore.json"
BARCODES = "/pbi/dept/secondary/siv/barcodes/RSII_3_barcodes/RSII_3_barcodes.barcodeset.xml"
SUBREADS_IN = "/pbi/dept/secondary/siv/testdata/SA3-Sequel/phi29/315/3150101/r54008_20160219_002905/1_A01_micro/m54008_160219_003234_micro_split.subreadset.xml"
report = run_to_report(SUBREADS, BARCODES, SUBREADS_IN)
validate_report_complete(self, report)
d = report.to_dict()
self.assertIsNotNone(d)
attr = {a.id: a.value for a in report.attributes}
self.assertEqual(attr['n_barcodes'], 3)
self.assertEqual(report.tables[0].columns[2].values,
['lbc1--lbc1', 'lbc2--lbc2', 'lbc3--lbc3', 'Not Barcoded'])
self.assertEqual(report.tables[0].columns[3].values, [2, 2, 1, 1])
def test_make_sv_report_table_empty_data(self):
table = op.join(_DATA_DIR, 'svann_empty.json')
plot = op.join(_DATA_DIR, 'svlengths_empty.json')
rpt = to_report(table, plot, self._output_dir)
d = json.loads(rpt.to_json())
self.assertTrue(op.exists(op.join(self._output_dir, 'sv_plot.png')))
self.assertTrue(
op.exists(op.join(self._output_dir, 'sv_plot_thumb.png')))
validate_report_complete(self, rpt)
def test_basic(self):
cmd = _to_cmd(self.ALIGNMENTS, self.report_json)
rcode = run_backticks(cmd)
self.assertEqual(rcode, 0)
with open(self.report_json, 'r') as f:
s = json.load(f)
log.info("JsonReport: ")
log.info(pprint.pformat(s, indent=4))
report = dict_to_report(s)
self.assertIsNotNone(report)
self.assertEqual(len(report.tables), 1)
log.info(str(report.tables[0]))
validate_report_metadata(self, report, spec)
validate_report_complete(self, report)
def test_basic(self):
cmd = _to_cmd(self.ALIGNMENTS, self.report_json)
rcode = run_backticks(cmd)
self.assertEqual(rcode, 0)
with open(self.report_json, 'r') as f:
s = json.load(f)
log.info("JsonReport: ")
log.info(pprint.pformat(s, indent=4))
report = dict_to_report(s)
self.assertIsNotNone(report)
self.assertEqual(len(report.tables), 1)
log.info(str(report.tables[0]))
validate_report_metadata(self, report, spec)
validate_report_complete(self, report)
def test_make_loading_report_with_dataset(self):
"""
Test the content of the loading report generated from a dataset
"""
sts_xml = data.getXmlWithStats()
rpt = make_loading_report(sts_xml)
d = json.loads(rpt.to_json())
t = d["tables"][0]
c0 = t["columns"][0]
c1 = t["columns"][1]
c2 = t["columns"][2]
c3 = t["columns"][3]
c4 = t["columns"][4]
self.assertEqual("Collection Context", c0["header"])
self.assertEqual("loading_xml_report.loading_xml_table." "collection_context", c0["id"])
self.assertEqual("Combined", c0["values"][0])
# Turn these back on when a context is inserted in the testdataset:
# self.assertEqual('dummy1', c0['values'][1])
# self.assertEqual('dummy2', c0['values'][2])
self.assertEqual("NA", c0["values"][1])
self.assertEqual("NA", c0["values"][2])
self.assertEqual("Productive ZMWs", c1["header"])
self.assertEqual("loading_xml_report.loading_xml_table." "productive_zmws", c1["id"])
self.assertEqual(153168, c1["values"][0])
self.assertEqual(2876, c1["values"][1])
self.assertEqual(150292, c1["values"][2])
self.assertEqual("Productivity 0 (%)", c2["header"])
self.assertEqual("loading_xml_report.loading_xml_table." "productivity_0", c2["id"])
self.assertAlmostEqual(78.985, c2["values"][0], delta=0.0003)
self.assertAlmostEqual(54.798, c2["values"][1], delta=0.0003)
self.assertAlmostEqual(79.447, c2["values"][2], delta=0.0003)
self.assertEqual("Productivity 1 (%)", c3["header"])
self.assertEqual("loading_xml_report.loading_xml_table." "productivity_1", c3["id"])
self.assertAlmostEqual(16.402, c3["values"][0], delta=0.0003)
self.assertAlmostEqual(31.328, c3["values"][1], delta=0.0003)
self.assertAlmostEqual(16.117, c3["values"][2], delta=0.0003)
self.assertEqual("Productivity 2 (%)", c4["header"])
self.assertEqual("loading_xml_report.loading_xml_table." "productivity_2", c4["id"])
self.assertAlmostEqual(4.613, c4["values"][0], delta=0.0003)
self.assertAlmostEqual(13.873, c4["values"][1], delta=0.0003)
self.assertAlmostEqual(4.436, c4["values"][2], delta=0.0003)
validate_report_complete(self, rpt)
def test_make_minor_topvariants_report(self):
"""
Call the main report generation method with minor kwarg==True. Deserialize report, check content.
"""
ref = self.REFERENCE
gff = self.RARE_VARIANTS_GFF
make_topvariants_report(
gff, ref, 100, 10000, 'rpt.json', self._output_dir, is_minor_variants_rpt=True)
# deserialize report
s = None
with open(os.path.join(self._output_dir, 'rpt.json'), 'r') as f:
s = json.load(f)
report = dict_to_report(s)
validate_report_complete(self, report)
def test_make_adapter_report_dataset(self):
"""
Test make_adapter_report with a dataset
"""
# All of the histogram generation code should be tested in
# pbcore.io.dataset, not here. Just test the report.
sts_xml = data.getXmlWithStats()
rpt = make_adapter_report(sts_xml, self.get_output_dir())
d = json.loads(rpt.to_json())
a = d["attributes"]
self.assertEqual(a[0]["name"], "Adapter Dimers (0-10bp) %")
self.assertEqual(a[0]["value"], 0.0)
self.assertEqual(a[1]["value"], 0.0)
self.assertTrue(os.path.exists(os.path.join(self.get_output_dir(), "interAdapterDist0.png")))
self.assertTrue(os.path.exists(os.path.join(self.get_output_dir(), "interAdapterDist1.png")))
validate_report_complete(self, rpt)
def test_make_polished_assembly_report(self):
"""
Test the attributes of the report
"""
make_polished_assembly_report(
'rpt.json', self.GFF, self.FASTQ, self._output_dir)
# deserialize report
s = None
with open(op.join(self._output_dir, 'rpt.json'), 'r') as f:
s = json.load(f)
self.report = dict_to_report(s)
self._test_attribute("polished_contigs")
self._test_attribute("max_contig_length")
self._test_attribute("n_50_contig_length")
self._test_attribute("sum_contig_lengths")
validate_report_complete(self, self.report)
def test_make_polished_assembly_report(self):
"""
Test the attributes of the report
"""
make_polished_assembly_report('rpt.json', self.GFF, self.FASTQ,
self._output_dir)
# deserialize report
s = None
with open(op.join(self._output_dir, 'rpt.json'), 'r') as f:
s = json.load(f)
self.report = dict_to_report(s)
self._test_attribute("polished_contigs")
self._test_attribute("max_contig_length")
self._test_attribute("n_50_contig_length")
self._test_attribute("sum_contig_lengths")
validate_report_complete(self, self.report)
def test_make_topvariants_report(self):
"""
Call the main report generation method. Deserialize report, check content.
"""
ref = self.REFERENCE
gff = self.VARIANTS_GFF
make_topvariants_report(gff, ref, 100, 10000, 'rpt.json',
self._output_dir)
# deserialize report
s = None
with open(os.path.join(self._output_dir, 'rpt.json'), 'r') as f:
s = json.load(f)
report = dict_to_report(s)
validate_report_complete(self, report)
def test_make_adapter_report_dataset2(self):
"""
Test make_adapter_report with a more recent dataset
"""
# All of the histogram generation code should be tested in
# pbcore.io.dataset, not here. Just test the report.
xml = pbtestdata.get_file("subreads-sequel")
rpt = make_adapter_report(xml, self.get_output_dir())
d = json.loads(rpt.to_json())
a = d['attributes']
self.assertEqual(a[0]['name'], 'Adapter Dimers (0-10bp) %')
self.assertEqual(a[0]['value'], 0.09)
self.assertEqual(a[1]['value'], 0.52)
self.assertEqual("{:.2f}".format(a[2]['value']), "0.00")
self.assertTrue(
os.path.exists(
os.path.join(self.get_output_dir(), 'interAdapterDist0.png')))
validate_report_complete(self, rpt)
def test_basic(self):
d = tempfile.mkdtemp()
t = tempfile.NamedTemporaryFile(
suffix="_motif_report.json", delete=False, dir=d)
t.close()
json_report = t.name
output_dir = os.path.dirname(json_report)
report = to_motifs_report(_MOTIF_GFF, _MOTIF_SUMMARY_CSV, output_dir)
validate_report_complete(self, report)
log.info(pformat(report.plotGroups[0].to_dict(), indent=4))
log.info(str(report.tables[0]))
self.assertTrue(True)
if os.path.exists(d):
shutil.rmtree(d)
def test_make_adapter_report_dataset(self):
"""
Test make_adapter_report with a dataset
"""
# All of the histogram generation code should be tested in
# pbcore.io.dataset, not here. Just test the report.
sts_xml = data.getXmlWithStats()
rpt = make_adapter_report(sts_xml, self.get_output_dir())
d = json.loads(rpt.to_json())
a = d['attributes']
self.assertEqual(a[0]['name'], 'Adapter Dimers (0-10bp) %')
self.assertEqual(a[0]['value'], 0.0)
self.assertEqual(a[1]['value'], 0.0)
self.assertTrue(
os.path.exists(
os.path.join(self.get_output_dir(), 'interAdapterDist0.png')))
self.assertTrue(
os.path.exists(
os.path.join(self.get_output_dir(), 'interAdapterDist1.png')))
validate_report_complete(self, rpt)
def test_basic(self):
d = tempfile.mkdtemp()
t = tempfile.NamedTemporaryFile(suffix="_motif_report.json",
delete=False,
dir=d)
t.close()
json_report = t.name
output_dir = os.path.dirname(json_report)
report = to_motifs_report(_MOTIF_GFF, _MOTIF_SUMMARY_CSV, output_dir)
validate_report_complete(self, report)
log.info(pformat(report.plotGroups[0].to_dict(), indent=4))
log.info(str(report.tables[0]))
self.assertTrue(True)
if os.path.exists(d):
shutil.rmtree(d)
def test_run_to_report(self):
report = run_to_report(self.subreads, self.barcodes, self.subreads)
validate_report_complete(self, report)
d = report.to_dict()
self.assertIsNotNone(d)
attr = {a.id: a.value for a in report.attributes}
self.assertEqual(attr, {
'mean_read_length': 5341,
'mean_longest_subread_length': 820,
'min_reads': 1,
'mean_reads': 1,
'n_barcodes': 2,
'max_reads': 1,
'n_barcoded_reads': 2,
'n_unbarcoded_reads': 1
})
self.assertEqual(report.tables[0].columns[2].values, [
'lbc1--lbc1', 'lbc3--lbc3', 'Not Barcoded'])
self.assertEqual(report.tables[0].columns[3].values, [1, 1, 1])
self.assertEqual(report.tables[0].columns[4].values, [1, 1, 1])
self.assertEqual(report.tables[0].columns[5].values, [1436, 204, 9791])
self.assertEqual(report.tables[0].columns[-1].values, [1, 2, None])
def test_exit_code_0(self):
bam = self.getAlignmentSet()
var_rpt = os.path.join(DATA, "variants_report.json")
mapping_rpt = os.path.join(DATA, "mapping_stats_report.json")
cmd = "python -m pbreports.report.sat {o} {r} {c} {a} {v}".format(
o=self._output_dir, r="rpt.json", c=bam, a=var_rpt, v=mapping_rpt
)
o, c, m = backticks(cmd)
log.info(cmd)
if c is not 0:
log.error(m)
log.error(o)
print(m)
self.assertEquals(0, c)
rpt_file = os.path.join(self._output_dir, "rpt.json")
rpt = load_report_from_json(rpt_file)
self.assertEqual("sidney", rpt.get_attribute_by_id("instrument").value)
self.assertEqual(1, rpt.get_attribute_by_id("coverage").value)
self.assertEqual(1, rpt.get_attribute_by_id("concordance").value)
self.assertEqual(7752, rpt.get_attribute_by_id("mapped_readlength_mean").value)
self.assertEqual(48, rpt.get_attribute_by_id("reads_in_cell").value)
out = StringIO()
self.assertTrue(summarize_report(rpt_file, out=out))
validate_report_complete(self, rpt)
def test_validate_report(self):
r = self._to_report()
self.assertIsInstance(r, Report)
validate_report_complete(self, r)
def test_make_sv_report(self):
table = op.join(_DATA_DIR, 'svann.json')
plot = op.join(_DATA_DIR, 'svlengths.json')
rpt = to_report(table, plot, self._output_dir)
d = json.loads(rpt.to_json())
self.assertEqual('Report structural_variants', d['title'])
t = d['tables'][1]
c0 = t['columns'][0]
c1 = t['columns'][1]
c2 = t['columns'][2]
c3 = t['columns'][3]
self.assertEqual('Count by Annotation', t['title'])
self.assertEqual('', c0['header'])
self.assertEqual('structural_variants.anno_table.annotation', c0['id'])
self.assertEqual('Alu', c0['values'][0])
self.assertEqual('Unannotated', c0['values'][1])
self.assertEqual('Total', c0['values'][2])
self.assertEqual('Insertions (total bp)', c1['header'])
self.assertEqual('structural_variants.anno_table.insertions', c1['id'])
self.assertEqual('1 (5)', c1['values'][0])
self.assertEqual('2 (21)', c1['values'][1])
self.assertEqual('3 (26)', c1['values'][2])
self.assertEqual('Deletions (total bp)', c2['header'])
self.assertEqual('structural_variants.anno_table.deletions', c2['id'])
self.assertEqual('0 (0)', c2['values'][0])
self.assertEqual('5 (9,898)', c2['values'][1])
self.assertEqual('5 (9,898)', c2['values'][2])
self.assertEqual('Total Variants (total bp)', c3['header'])
self.assertEqual('structural_variants.anno_table.total', c3['id'])
self.assertEqual('1 (5)', c3['values'][0])
self.assertEqual('7 (9,919)', c3['values'][1])
self.assertEqual('8 (9,924)', c3['values'][2])
t = d['tables'][0]
c0 = t['columns'][0]
c1 = t['columns'][1]
c2 = t['columns'][2]
c3 = t['columns'][3]
c4 = t['columns'][4]
c5 = t['columns'][5]
self.assertEqual('Count by Sample', t['title'])
self.assertEqual('', c0['header'])
self.assertEqual('structural_variants.sample_table.sample', c0['id'])
self.assertEqual('SAMPLE_1', c0['values'][0])
self.assertEqual('SAMPLE_2', c0['values'][1])
self.assertEqual('Insertions (total bp)', c1['header'])
self.assertEqual('structural_variants.sample_table.insertions',
c1['id'])
self.assertEqual('2 (423)', c1['values'][0])
self.assertEqual('2 (425)', c1['values'][1])
self.assertEqual('Deletions (total bp)', c2['header'])
self.assertEqual('structural_variants.sample_table.deletions',
c2['id'])
self.assertEqual('7 (17,771)', c2['values'][0])
self.assertEqual('7 (7,771)', c2['values'][1])
self.assertEqual('Homozygous Variants', c3['header'])
self.assertEqual('structural_variants.sample_table.homozygous',
c3['id'])
self.assertEqual(3, c3['values'][0])
self.assertEqual(3, c3['values'][1])
self.assertEqual('Heterozygous Variants', c4['header'])
self.assertEqual('structural_variants.sample_table.heterozygous',
c4['id'])
self.assertEqual(6, c4['values'][0])
self.assertEqual(5, c4['values'][1])
self.assertEqual('Total Variants (total bp)', c5['header'])
self.assertEqual('structural_variants.sample_table.total', c5['id'])
self.assertEqual('9 (18,194)', c5['values'][0])
self.assertEqual('10 (8,196)', c5['values'][1])
self.assertTrue(op.exists(op.join(self._output_dir, 'sv_plot.png')))
self.assertTrue(
op.exists(op.join(self._output_dir, 'sv_plot_thumb.png')))
validate_report_complete(self, rpt)
def test_make_loading_report_with_merged_dataset(self):
"""
Test the content of the loading report generated from a dataset with
multiple sub-datasets
"""
ss = get_merged_subreadset()
rpt = make_loading_report(ss, self.get_output_dir())
d = json.loads(rpt.to_json())
t = d['tables'][0]
c0 = t['columns'][0]
c1 = t['columns'][1]
c2 = t['columns'][2]
c3 = t['columns'][3]
c4 = t['columns'][4]
c5 = t['columns'][5]
c6 = t['columns'][6]
c7 = t['columns'][7]
self.assertEqual('Collection Context', c0['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'collection_context', c0['id'])
self.assertEqual('Combined', c0['values'][0])
# Turn these back on when a context is inserted in the testdataset:
#self.assertEqual('dummy1', c0['values'][1])
#self.assertEqual('dummy2', c0['values'][2])
self.assertEqual('m54086_160831_230338', c0['values'][1])
self.assertEqual('m54099_160830_232701', c0['values'][2])
self.assertEqual('Productive ZMWs', c1['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'productive_zmws', c1['id'])
self.assertEqual(2073466, c1['values'][0])
self.assertEqual(1036722, c1['values'][1])
self.assertEqual(1036744, c1['values'][2])
self.assertEqual('Productivity 0', c2['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'productivity_0_n', c2['id'])
self.assertEqual(1325603, c2['values'][0])
self.assertEqual(853636, c2['values'][1])
self.assertEqual(471967, c2['values'][2])
self.assertEqual('(%)', c3['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'productivity_0_pct', c3['id'])
self.assertAlmostEqual(63.932, c3['values'][0], delta=.0003)
self.assertAlmostEqual(82.340, c3['values'][1], delta=.0003)
self.assertAlmostEqual(45.524, c3['values'][2], delta=.0003)
self.assertEqual('Productivity 1', c4['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'productivity_1_n', c4['id'])
self.assertEqual(743972, c4['values'][0])
self.assertEqual(180176, c4['values'][1])
self.assertEqual(563796, c4['values'][2])
self.assertEqual('(%)', c5['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'productivity_1_pct', c5['id'])
self.assertAlmostEqual(35.881, c5['values'][0], delta=.0003)
self.assertAlmostEqual(17.379, c5['values'][1], delta=.0003)
self.assertAlmostEqual(54.381, c5['values'][2], delta=.0003)
self.assertEqual('Productivity 2', c6['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'productivity_2_n', c6['id'])
self.assertEqual(3891, c6['values'][0])
self.assertEqual(2910, c6['values'][1])
self.assertEqual(981, c6['values'][2])
self.assertEqual('(%)', c7['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'productivity_2_pct', c7['id'])
self.assertAlmostEqual(0.188, c7['values'][0], delta=.0003)
self.assertAlmostEqual(0.281, c7['values'][1], delta=.0003)
self.assertAlmostEqual(0.095, c7['values'][2], delta=.0003)
self.assertTrue(
os.path.exists(
os.path.join(self.get_output_dir(),
'raw_read_length_plot.png')))
self.assertTrue(
os.path.exists(
os.path.join(self.get_output_dir(),
'raw_read_length_plot_thumb.png')))
validate_report_complete(self, rpt)
def test_basic(self):
report = run_to_report(self.file_name)
validate_report_complete(self, report)
self.assertTrue(isinstance(report, Report))
log.info(pformat(report.to_dict()))
self.assertIsNotNone(report)
def test_basic(self):
self.assertIsNotNone(self.report)
validate_report_complete(self, self.report)
def test_basic(self):
self.assertIsNotNone(self.report)
validate_report_complete(self, self.report)
def test_validate_report(self):
r = self._to_report()
self.assertIsInstance(r, Report)
validate_report_complete(self, r)
def test_make_loading_report_with_dataset(self):
"""
Test the content of the loading report generated from a simple
SubreadSet
"""
sts_xml = pbtestdata.get_file("subreads-sequel")
rpt = make_loading_report(sts_xml, self.get_output_dir())
d = json.loads(rpt.to_json())
t = d['tables'][0]
c0 = t['columns'][0]
c1 = t['columns'][1]
c2 = t['columns'][2]
c3 = t['columns'][3]
c4 = t['columns'][4]
c5 = t['columns'][5]
c6 = t['columns'][6]
c7 = t['columns'][7]
c8 = t['columns'][8]
self.assertEqual('Collection Context', c0['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'collection_context', c0['id'])
self.assertEqual('m54006_160504_020705', c0['values'][0])
self.assertEqual('Productive ZMWs', c1['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'productive_zmws', c1['id'])
self.assertEqual(1035426, c1['values'][0])
self.assertEqual('Productivity 0', c2['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'productivity_0_n', c2['id'])
self.assertEqual(631530, c2['values'][0])
self.assertEqual('(%)', c3['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'productivity_0_pct', c3['id'])
self.assertAlmostEqual(60.992, c3['values'][0], delta=.0003)
self.assertEqual('Productivity 1', c4['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'productivity_1_n', c4['id'])
self.assertEqual(394658, c4['values'][0])
self.assertEqual('(%)', c5['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'productivity_1_pct', c5['id'])
self.assertAlmostEqual(38.116, c5['values'][0], delta=.0003)
self.assertEqual('Productivity 2', c6['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'productivity_2_n', c6['id'])
self.assertEqual(9238, c6['values'][0])
self.assertEqual('(%)', c7['header'])
self.assertEqual(
'loading_xml_report.loading_xml_table.'
'productivity_2_pct', c7['id'])
self.assertAlmostEqual(0.892, c7['values'][0], delta=.0003)
self.assertEqual(c8['values'][0], "Workflow_Magbead.py")
self.assertTrue(
os.path.exists(
os.path.join(self.get_output_dir(),
'raw_read_length_plot.png')))
self.assertTrue(
os.path.exists(
os.path.join(self.get_output_dir(),
'raw_read_length_plot_thumb.png')))
validate_report_complete(self, rpt)
def test_basic(self):
report = run_to_report(self.input_log_file)
validate_report_complete(self, report)
self.assertTrue(isinstance(report, Report))
log.info(pformat(report.to_dict()))
self.assertIsNotNone(report)
