def test_make_adapter_report_sts_xml(self):
        """
        Test make_adapter_report with an sts.xml file
        """
        # All of the histogram generation code should be tested in
        # pbcore.io.dataset, not here. Just test the report.
        try:
            log.info(
                TestXMLstatsRpts.test_make_adapter_report_sts_xml.__doc__)
            sts_xml = data.getStats(1)
            rpt = make_adapter_report(sts_xml, self.get_output_dir())

            d = json.loads(rpt.to_json())

            t = d['tables'][0]
            c0 = t['columns'][0]
            c1 = t['columns'][1]

            self.assertEqual('Adapter Dimers (0-10bp)', c0['values'][0])
            self.assertEqual('Short Inserts (11-100bp)', c0['values'][1])
            self.assertEqual(0.0, c1['values'][0])
            self.assertEqual(0.0, c1['values'][1])
            self.assertTrue(os.path.exists(os.path.join(
                self.get_output_dir(),
                'interAdapterDist0.png')))

        except:
            log.error(traceback.format_exc())
            raise
    def test_make_loading_report_with_sts_xml(self):
        """
        Test the content of the loading report generated from a sts.xml
        """
        sts_xml = data.getStats(0)

        rpt = make_loading_report(sts_xml)
        d = json.loads(rpt.to_json())

        t = d["tables"][0]
        c0 = t["columns"][0]
        c1 = t["columns"][1]
        c2 = t["columns"][2]
        c3 = t["columns"][3]
        c4 = t["columns"][4]

        self.assertEqual("Collection Context", c0["header"])
        self.assertEqual("loading_xml_report.loading_xml_table." "collection_context", c0["id"])
        self.assertEqual("NA", c0["values"][0])

        self.assertEqual("Productive ZMWs", c1["header"])
        self.assertEqual("loading_xml_report.loading_xml_table." "productive_zmws", c1["id"])
        self.assertEqual(2876, c1["values"][0])

        self.assertEqual("Productivity 0 (%)", c2["header"])
        self.assertEqual("loading_xml_report.loading_xml_table." "productivity_0", c2["id"])
        self.assertAlmostEqual(54.798, c2["values"][0], delta=0.0003)

        self.assertEqual("Productivity 1 (%)", c3["header"])
        self.assertEqual("loading_xml_report.loading_xml_table." "productivity_1", c3["id"])
        self.assertAlmostEqual(31.328, c3["values"][0], delta=0.0003)

        self.assertEqual("Productivity 2 (%)", c4["header"])
        self.assertEqual("loading_xml_report.loading_xml_table." "productivity_2", c4["id"])
        self.assertAlmostEqual(13.873, c4["values"][0], delta=0.0003)
示例#3
0
 def test_loadstats_cli(self):
     outfile = os.path.join(tempfile.mkdtemp(suffix="dataset-unittest"),
                            'withStats.alignmentset.xml')
     cmd = "dataset loadstats {d} {s} --outfile {o}".format(
         o=outfile, d=data.getXml(7), s=data.getStats())
     self._run_cmd_with_output(cmd, outfile)
     aln = AlignmentSet(outfile)
     assert aln.metadata.summaryStats
示例#4
0
 def test_loadstats_cli(self):
     outfile = os.path.join(tempfile.mkdtemp(suffix="dataset-unittest"),
                            'withStats.alignmentset.xml')
     cmd = "dataset loadstats {d} {s} --outfile {o}".format(
         o=outfile, d=data.getXml(8), s=data.getStats())
     log.debug(cmd)
     o, r, m = backticks(cmd)
     self.assertEqual(r, 0)
     self.assertTrue(os.path.exists(outfile))
     aln = AlignmentSet(outfile)
     self.assertTrue(aln.metadata.summaryStats)
示例#5
0
 def test_loadstats_cli(self):
     outfile = os.path.join(
         tempfile.mkdtemp(suffix="dataset-unittest"),
         'withStats.alignmentset.xml')
     cmd = "dataset loadstats {d} {s} --outfile {o}".format(
         o=outfile, d=data.getXml(8), s=data.getStats())
     log.debug(cmd)
     o, r, m = backticks(cmd)
     self.assertEqual(r, 0)
     self.assertTrue(os.path.exists(outfile))
     aln = AlignmentSet(outfile)
     self.assertTrue(aln.metadata.summaryStats)
    def test_make_loading_report_with_sts_xml(self):
        """
        Test the content of the loading report generated from a sts.xml
        """
        try:
            log.info(
                TestXMLstatsRpts.test_make_loading_report_with_sts_xml.__doc__)
            sts_xml = data.getStats(0)

            rpt = make_loading_report(sts_xml)
            d = json.loads(rpt.to_json())

            t = d['tables'][0]
            c0 = t['columns'][0]
            c1 = t['columns'][1]
            c2 = t['columns'][2]
            c3 = t['columns'][3]
            c4 = t['columns'][4]

            self.assertEqual('Collection Context', c0['header'])
            self.assertEqual('loading_xml_report.loading_xml_table.'
                             'collection_context', c0['id'])
            self.assertEqual('NA', c0['values'][0])

            self.assertEqual('Productive ZMWs', c1['header'])
            self.assertEqual('loading_xml_report.loading_xml_table.'
                             'productive_zmws', c1['id'])
            self.assertEqual(2876, c1['values'][0])

            self.assertEqual('Productivity 0 (%)',
                             c2['header'])
            self.assertEqual('loading_xml_report.loading_xml_table.'
                             'productivity_0',
                             c2['id'])
            self.assertAlmostEqual(54.80, c2['values'][0], delta=.0003)

            self.assertEqual('Productivity 1 (%)',
                             c3['header'])
            self.assertEqual('loading_xml_report.loading_xml_table.'
                             'productivity_1',
                             c3['id'])
            self.assertAlmostEqual(31.33, c3['values'][0], delta=.0003)

            self.assertEqual('Productivity 2 (%)',
                             c4['header'])
            self.assertEqual('loading_xml_report.loading_xml_table.'
                             'productivity_2',
                             c4['id'])
            self.assertAlmostEqual(13.87, c4['values'][0], delta=.0003)

        except:
            log.error(traceback.format_exc())
            raise
 def test_adapter_exit_code_0(self):
     log.info(TestXMLstatsRpts.test_adapter_exit_code_0.__doc__)
     sts_xml = data.getStats(0)
     cmd = 'adapter_xml {c} {r}'.format(
         r='foo.json',
         c=sts_xml)
     o, c, m = backticks(cmd)
     print "COMMAND: {c}".format(c=cmd)
     log.info(cmd)
     print "o: {o}".format(o=o)
     print "c: {c}".format(c=c)
     print "m: {m}".format(m=m)
     if c is not 0:
         log.error(m)
         log.error(o)
     self.assertEquals(0, c)
    def test_make_filter_stats_report_sts_xml(self):
        """
        Test the content of the filter report generated from an sts.xml
        """
        try:
            log.info(
                TestXMLstatsRpts.test_make_filter_stats_report_sts_xml.__doc__)
            sts_xml = data.getStats(0)

            rpt = make_filter_report(sts_xml, self.get_output_dir())

            d = json.loads(rpt.to_json())

            t = d['tables'][0]
            c0 = t['columns'][0]
            c1 = t['columns'][1]
            self.assertEqual('Metrics', c0['header'])
            self.assertEqual('filtering_stats_xml_report.filter_xml_table.'
                             'filter_names_column', c0['id'])
            self.assertEqual('Polymerase Read Bases', c0['values'][0])
            self.assertEqual('Polymerase Reads', c0['values'][1])
            self.assertEqual('Polymerase Read N50', c0['values'][2])
            self.assertEqual('Polymerase Read Length', c0['values'][3])
            self.assertEqual('Polymerase Read Quality', c0['values'][4])

            self.assertEqual('Values', c1['header'])
            self.assertEqual('filtering_stats_xml_report.filter_xml_table.'
                             'filter_stats_column', c1['id'])
            self.assertEqual(4464266.29, c1['values'][0])
            self.assertEqual(901, c1['values'][1])
            self.assertEqual(6570, c1['values'][2])
            self.assertEqual(4954.79, c1['values'][3])
            self.assertEqual(0.83, c1['values'][4])
            self.assertTrue(os.path.exists(os.path.join(
                self.get_output_dir(),
                'readLenDist0.png')))
            self.assertTrue(os.path.exists(os.path.join(
                self.get_output_dir(),
                'readQualDist0.png')))

            # these are from a raw STS file
            self.assertEqual(len(rpt._dataset_uuids), 0,
                             "Incorrect report datasets uuids")

        except:
            log.error(traceback.format_exc())
            raise
    def test_make_loading_report_with_sts_xml(self):
        """
        Test the content of the loading report generated from a sts.xml
        """
        sts_xml = data.getStats(0)

        rpt = make_loading_report(sts_xml)
        d = json.loads(rpt.to_json())

        t = d['tables'][0]
        c0 = t['columns'][0]
        c1 = t['columns'][1]
        c2 = t['columns'][2]
        c3 = t['columns'][3]
        c4 = t['columns'][4]

        self.assertEqual('Collection Context', c0['header'])
        self.assertEqual('loading_xml_report.loading_xml_table.'
                         'collection_context', c0['id'])
        self.assertEqual('NA', c0['values'][0])

        self.assertEqual('Productive ZMWs', c1['header'])
        self.assertEqual('loading_xml_report.loading_xml_table.'
                         'productive_zmws', c1['id'])
        self.assertEqual(2876, c1['values'][0])

        self.assertEqual('Productivity 0 (%)',
                         c2['header'])
        self.assertEqual('loading_xml_report.loading_xml_table.'
                         'productivity_0',
                         c2['id'])
        self.assertAlmostEqual(54.798, c2['values'][0], delta=.0003)

        self.assertEqual('Productivity 1 (%)',
                         c3['header'])
        self.assertEqual('loading_xml_report.loading_xml_table.'
                         'productivity_1',
                         c3['id'])
        self.assertAlmostEqual(31.328, c3['values'][0], delta=.0003)

        self.assertEqual('Productivity 2 (%)',
                         c4['header'])
        self.assertEqual('loading_xml_report.loading_xml_table.'
                         'productivity_2',
                         c4['id'])
        self.assertAlmostEqual(13.873, c4['values'][0], delta=.0003)
示例#10
0
    def test_make_loading_report_with_sts_xml(self):
        """
        Test the content of the loading report generated from a sts.xml
        """
        sts_xml = data.getStats(0)

        rpt = make_loading_report(sts_xml)
        d = json.loads(rpt.to_json())

        t = d['tables'][0]
        c0 = t['columns'][0]
        c1 = t['columns'][1]
        c2 = t['columns'][2]
        c3 = t['columns'][3]
        c4 = t['columns'][4]

        self.assertEqual('Collection Context', c0['header'])
        self.assertEqual(
            'loading_xml_report.loading_xml_table.'
            'collection_context', c0['id'])
        self.assertEqual('NA', c0['values'][0])

        self.assertEqual('Productive ZMWs', c1['header'])
        self.assertEqual(
            'loading_xml_report.loading_xml_table.'
            'productive_zmws', c1['id'])
        self.assertEqual(2876, c1['values'][0])

        self.assertEqual('Productivity 0 (%)', c2['header'])
        self.assertEqual(
            'loading_xml_report.loading_xml_table.'
            'productivity_0', c2['id'])
        self.assertAlmostEqual(54.798, c2['values'][0], delta=.0003)

        self.assertEqual('Productivity 1 (%)', c3['header'])
        self.assertEqual(
            'loading_xml_report.loading_xml_table.'
            'productivity_1', c3['id'])
        self.assertAlmostEqual(31.328, c3['values'][0], delta=.0003)

        self.assertEqual('Productivity 2 (%)', c4['header'])
        self.assertEqual(
            'loading_xml_report.loading_xml_table.'
            'productivity_2', c4['id'])
        self.assertAlmostEqual(13.873, c4['values'][0], delta=.0003)
示例#11
0
    def test_stats_metadata(self):
        ds = DataSet(data.getBam())
        ds.loadStats(data.getStats())
        self.assertEqual(ds.metadata.summaryStats.prodDist.numBins, 4)
        self.assertEqual(ds.metadata.summaryStats.prodDist.bins,
                         [1576, 901, 399, 0])
        ds1 = DataSet(data.getXml(8))
        ds1.loadStats(data.getStats())
        ds2 = DataSet(data.getXml(11))
        ds2.loadStats(data.getStats())
        ds3 = ds1 + ds2
        self.assertEqual(ds1.metadata.summaryStats.prodDist.bins,
                         [1576, 901, 399, 0])
        self.assertEqual(ds2.metadata.summaryStats.prodDist.bins,
                         [1576, 901, 399, 0])
        self.assertEqual(ds3.metadata.summaryStats.prodDist.bins,
                         [3152, 1802, 798, 0])
        self.assertEqual(ds1.metadata.summaryStats.readLenDist.bins,
                         [0, 62, 39, 36, 29, 37, 19, 29, 37, 32, 32, 40, 45,
                          54, 73, 77, 97, 95, 49, 17, 2, 0, 0, 0, 0, 0, 0,
                          0, 0, 0, 0, 0])
        self.assertEqual(ds2.metadata.summaryStats.readLenDist.bins,
                         [0, 62, 39, 36, 29, 37, 19, 29, 37, 32, 32, 40, 45,
                          54, 73, 77, 97, 95, 49, 17, 2, 0, 0, 0, 0, 0, 0,
                          0, 0, 0, 0, 0])
        self.assertEqual(ds3.metadata.summaryStats.readLenDist.bins,
                         [0, 124, 78, 72, 58, 74, 38, 58, 74, 64, 64, 80, 90,
                          108, 146, 154, 194, 190, 98, 34, 4, 0, 0, 0, 0, 0,
                          0, 0, 0, 0, 0, 0])
        # Lets check some manual values
        ds1 = DataSet(data.getXml(8))
        ds1.loadStats(data.getStats())
        ds2 = DataSet(data.getXml(11))
        ds2.loadStats(data.getStats())
        ds1.metadata.summaryStats.readLenDist.bins = (
            [0, 10, 9, 8, 7, 6, 4, 2, 1, 0, 0, 1])
        self.assertEqual(ds1.metadata.summaryStats.readLenDist.bins,
                         [0, 10, 9, 8, 7, 6, 4, 2, 1, 0, 0, 1])
        ds1.metadata.summaryStats.readLenDist.minBinValue = 10
        ds1.metadata.summaryStats.readLenDist.binWidth = 10
        ds2.metadata.summaryStats.readLenDist.bins = (
            [0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1])
        self.assertEqual(ds2.metadata.summaryStats.readLenDist.bins,
                         [0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1])
        ds2.metadata.summaryStats.readLenDist.minBinValue = 20
        ds2.metadata.summaryStats.readLenDist.binWidth = 10
        ds3 = ds1 + ds2
        self.assertEqual(ds3.metadata.summaryStats.readLenDist.bins,
                         [0, 10, 10, 9, 8, 7, 5, 3, 2, 1, 0, 1, 1])
        # now lets swap
        ds1 = DataSet(data.getXml(8))
        ds1.loadStats(data.getStats())
        ds2 = DataSet(data.getXml(11))
        ds2.loadStats(data.getStats())
        ds1.metadata.summaryStats.readLenDist.bins = (
            [0, 10, 9, 8, 7, 6, 4, 2, 1, 0, 0, 1])
        self.assertEqual(ds1.metadata.summaryStats.readLenDist.bins,
                         [0, 10, 9, 8, 7, 6, 4, 2, 1, 0, 0, 1])
        ds1.metadata.summaryStats.readLenDist.minBinValue = 20
        ds1.metadata.summaryStats.readLenDist.binWidth = 10
        ds2.metadata.summaryStats.readLenDist.bins = (
            [0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1])
        self.assertEqual(ds2.metadata.summaryStats.readLenDist.bins,
                         [0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1])
        ds2.metadata.summaryStats.readLenDist.minBinValue = 10
        ds2.metadata.summaryStats.readLenDist.binWidth = 10
        ds3 = ds1 + ds2
        self.assertEqual(ds3.metadata.summaryStats.readLenDist.bins,
                         [0, 1, 11, 10, 9, 8, 7, 5, 3, 1, 0, 1, 1])

        # now lets do some non-overlapping
        ds1 = DataSet(data.getXml(8))
        ds1.loadStats(data.getStats())
        ds2 = DataSet(data.getXml(11))
        ds2.loadStats(data.getStats())
        ds1.metadata.summaryStats.readLenDist.bins = (
            [1, 1, 1])
        self.assertEqual(ds1.metadata.summaryStats.readLenDist.bins,
                         [1, 1, 1])
        ds1.metadata.summaryStats.readLenDist.minBinValue = 10
        ds1.metadata.summaryStats.readLenDist.binWidth = 10
        ds2.metadata.summaryStats.readLenDist.bins = (
            [2, 2, 2])
        self.assertEqual(ds2.metadata.summaryStats.readLenDist.bins,
                         [2, 2, 2])
        ds2.metadata.summaryStats.readLenDist.minBinValue = 50
        ds2.metadata.summaryStats.readLenDist.binWidth = 10
        ds3 = ds1 + ds2
        self.assertEqual(ds3.metadata.summaryStats.readLenDist.bins,
                         [1, 1, 1, 0, 2, 2, 2])

        # now lets test the subdataset metadata retention:
        ss = SubreadSet(data.getXml(10))
        ss.loadStats(data.getStats(0))
        ss.loadStats(data.getStats(1))
        self.assertEqual(153168.0, ss.metadata.summaryStats.numSequencingZmws)
        self.assertEqual(
            2876.0, ss.subdatasets[0].metadata.summaryStats.numSequencingZmws)
        self.assertEqual(
            150292.0, ss.subdatasets[1].metadata.summaryStats.numSequencingZmws)
示例#12
0
    def test_stats_metadata(self):
        ds = DataSet(data.getBam())
        ds.loadStats(data.getStats())
        self.assertEqual(ds.metadata.summaryStats.prodDist.numBins, 4)
        self.assertEqual(ds.metadata.summaryStats.prodDist.bins,
                         [1576, 901, 399, 0])
        ds1 = DataSet(data.getXml(8))
        ds1.loadStats(data.getStats())
        ds2 = DataSet(data.getXml(11))
        ds2.loadStats(data.getStats())
        ds3 = ds1 + ds2
        self.assertEqual(ds1.metadata.summaryStats.prodDist.bins,
                         [1576, 901, 399, 0])
        self.assertEqual(ds2.metadata.summaryStats.prodDist.bins,
                         [1576, 901, 399, 0])
        self.assertEqual(ds3.metadata.summaryStats.prodDist.bins,
                         [3152, 1802, 798, 0])
        self.assertEqual(ds1.metadata.summaryStats.readLenDist.bins, [
            0, 62, 39, 36, 29, 37, 19, 29, 37, 32, 32, 40, 45, 54, 73, 77, 97,
            95, 49, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
        ])
        self.assertEqual(ds2.metadata.summaryStats.readLenDist.bins, [
            0, 62, 39, 36, 29, 37, 19, 29, 37, 32, 32, 40, 45, 54, 73, 77, 97,
            95, 49, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
        ])
        self.assertEqual(ds3.metadata.summaryStats.readLenDist.bins, [
            0, 124, 78, 72, 58, 74, 38, 58, 74, 64, 64, 80, 90, 108, 146, 154,
            194, 190, 98, 34, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
        ])
        # Lets check some manual values
        ds1 = DataSet(data.getXml(8))
        ds1.loadStats(data.getStats())
        ds2 = DataSet(data.getXml(11))
        ds2.loadStats(data.getStats())
        ds1.metadata.summaryStats.readLenDist.bins = ([
            0, 10, 9, 8, 7, 6, 4, 2, 1, 0, 0, 1
        ])
        self.assertEqual(ds1.metadata.summaryStats.readLenDist.bins,
                         [0, 10, 9, 8, 7, 6, 4, 2, 1, 0, 0, 1])
        ds1.metadata.summaryStats.readLenDist.minBinValue = 10
        ds1.metadata.summaryStats.readLenDist.binWidth = 10
        ds2.metadata.summaryStats.readLenDist.bins = ([
            0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1
        ])
        self.assertEqual(ds2.metadata.summaryStats.readLenDist.bins,
                         [0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1])
        ds2.metadata.summaryStats.readLenDist.minBinValue = 20
        ds2.metadata.summaryStats.readLenDist.binWidth = 10
        ds3 = ds1 + ds2
        self.assertEqual(ds3.metadata.summaryStats.readLenDist.bins,
                         [0, 10, 10, 9, 8, 7, 5, 3, 2, 1, 0, 1, 1])
        # now lets swap
        ds1 = DataSet(data.getXml(8))
        ds1.loadStats(data.getStats())
        ds2 = DataSet(data.getXml(11))
        ds2.loadStats(data.getStats())
        ds1.metadata.summaryStats.readLenDist.bins = ([
            0, 10, 9, 8, 7, 6, 4, 2, 1, 0, 0, 1
        ])
        self.assertEqual(ds1.metadata.summaryStats.readLenDist.bins,
                         [0, 10, 9, 8, 7, 6, 4, 2, 1, 0, 0, 1])
        ds1.metadata.summaryStats.readLenDist.minBinValue = 20
        ds1.metadata.summaryStats.readLenDist.binWidth = 10
        ds2.metadata.summaryStats.readLenDist.bins = ([
            0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1
        ])
        self.assertEqual(ds2.metadata.summaryStats.readLenDist.bins,
                         [0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1])
        ds2.metadata.summaryStats.readLenDist.minBinValue = 10
        ds2.metadata.summaryStats.readLenDist.binWidth = 10
        ds3 = ds1 + ds2
        self.assertEqual(ds3.metadata.summaryStats.readLenDist.bins,
                         [0, 1, 11, 10, 9, 8, 7, 5, 3, 1, 0, 1, 1])

        # now lets do some non-overlapping
        ds1 = DataSet(data.getXml(8))
        ds1.loadStats(data.getStats())
        ds2 = DataSet(data.getXml(11))
        ds2.loadStats(data.getStats())
        ds1.metadata.summaryStats.readLenDist.bins = ([1, 1, 1])
        self.assertEqual(ds1.metadata.summaryStats.readLenDist.bins, [1, 1, 1])
        ds1.metadata.summaryStats.readLenDist.minBinValue = 10
        ds1.metadata.summaryStats.readLenDist.binWidth = 10
        ds2.metadata.summaryStats.readLenDist.bins = ([2, 2, 2])
        self.assertEqual(ds2.metadata.summaryStats.readLenDist.bins, [2, 2, 2])
        ds2.metadata.summaryStats.readLenDist.minBinValue = 50
        ds2.metadata.summaryStats.readLenDist.binWidth = 10
        ds3 = ds1 + ds2
        self.assertEqual(ds3.metadata.summaryStats.readLenDist.bins,
                         [1, 1, 1, 0, 2, 2, 2])

        # now lets test the subdataset metadata retention:
        ss = SubreadSet(data.getXml(10))
        ss.loadStats(data.getStats(0))
        ss.loadStats(data.getStats(1))
        self.assertEqual(153168.0, ss.metadata.summaryStats.numSequencingZmws)
        self.assertEqual(
            2876.0, ss.subdatasets[0].metadata.summaryStats.numSequencingZmws)
        self.assertEqual(
            150292.0,
            ss.subdatasets[1].metadata.summaryStats.numSequencingZmws)