Ejemplo n.º 1
0
    def test_generate_tfrecords(self):
        examples_out = os.path.join(absltest.get_default_test_tmpdir(),
                                    'examples_output')
        train_test_val_split = [0.7, 0.2, 0.1]
        ngs_read_length = ngs_errors.generate_tfrecord_datasets(
            train_test_val_split,
            ref_path=test_utils.genomics_core_testdata(
                'ucsc.hg19.chr20.unittest.fasta.gz'),
            vcf_path=test_utils.genomics_core_testdata(
                'test_nist.b37_chr20_100kbp_at_10mb.vcf.gz'),
            bam_path=test_utils.genomics_core_testdata(
                'NA12878_S1.chr20.10_10p1mb.bam'),
            out_dir=examples_out,
            max_reads=100)

        actual_examples = self._read_examples(train_test_val_split,
                                              examples_out)
        golden_examples = self._read_examples(
            train_test_val_split,
            test_utils.genomics_core_testdata('golden.examples.ngs_errors'))
        self.assertEqual(len(actual_examples), len(golden_examples))

        matched_examples = []
        for expected in golden_examples:
            for actual in actual_examples:
                if all(actual.features.feature[key] ==
                       expected.features.feature[key]
                       for key in expected.features.feature.keys()):
                    matched_examples.append(expected)
        self.assertEqual(golden_examples, matched_examples)
Ejemplo n.º 2
0
    def test_wrap(self, fasta_filename):
        chr_names = ['chrM', 'chr1', 'chr2']
        chr_lengths = [100, 76, 121]
        fasta = test_utils.genomics_core_testdata(fasta_filename)
        fai = test_utils.genomics_core_testdata(fasta_filename + '.fai')
        with reference_fai.GenomeReferenceFai.from_file(fasta, fai) as ref:
            self.assertEqual(ref.n_contigs, 3)
            self.assertIn(fasta, ref.fasta_path)
            self.assertIn('GenomeReference backed by htslib FAI index',
                          str(ref))
            self.assertEqual(ref.contig_names, chr_names)
            self.assertEqual(ref.n_bp, sum(chr_lengths))
            self.assertEqual(ref.bases(ranges.make_range('chrM', 1, 10)),
                             'ATCACAGGT')

            self.assertTrue(
                ref.is_valid_interval(ranges.make_range('chrM', 1, 10)))
            self.assertFalse(
                ref.is_valid_interval(ranges.make_range('chrM', 1, 100000)))

            self.assertEqual(len(ref.contigs), 3)
            self.assertEqual([c.name for c in ref.contigs], chr_names)
            self.assertEqual([c.n_bases for c in ref.contigs], chr_lengths)
            for contig in ref.contigs:
                self.assertEqual(ref.contig(contig.name), contig)
                self.assertTrue(ref.has_contig(contig.name))
                self.assertFalse(ref.has_contig(contig.name + '.unknown'))
Ejemplo n.º 3
0
    def test_main(self):
        examples_out = test_utils.test_tmpfile('output.tfrecord')
        ngs_errors.make_ngs_error_examples(
            ref_path=test_utils.genomics_core_testdata(
                'ucsc.hg19.chr20.unittest.fasta.gz'),
            vcf_path=test_utils.genomics_core_testdata(
                'test_nist.b37_chr20_100kbp_at_10mb.vcf.gz'),
            bam_path=test_utils.genomics_core_testdata(
                'NA12878_S1.chr20.10_10p1mb.bam'),
            examples_out_path=examples_out,
            max_reads=100)

        actual_examples = _read_examples(examples_out)
        golden_examples = _read_examples(
            test_utils.genomics_core_testdata(
                'golden.examples.ngs_errors.tfrecord'))

        self.assertEqual(len(actual_examples), 100)
        assertExamplesAreEqual(self,
                               golden_examples,
                               actual_examples,
                               expected_keys={
                                   'read_name', 'cigar', 'read_sequence',
                                   'read_qualities', 'true_sequence'
                               })
Ejemplo n.º 4
0
 def setUp(self):
   self.sites_vcf = test_utils.genomics_core_testdata('test_sites.vcf')
   self.samples_vcf = test_utils.genomics_core_testdata('test_samples.vcf.gz')
   self.options = variants_pb2.VcfReaderOptions()
   self.sites_reader = vcf_reader.VcfReader.from_file(self.sites_vcf,
                                                      self.options)
   self.samples_reader = vcf_reader.VcfReader.from_file(
       self.samples_vcf, self.options)
Ejemplo n.º 5
0
    def setUp(self):
        self.sites_reader = vcf.VcfReader(
            test_utils.genomics_core_testdata('test_sites.vcf'),
            use_index=False)

        self.samples_reader = vcf.VcfReader(
            test_utils.genomics_core_testdata('test_samples.vcf.gz'),
            use_index=True)
Ejemplo n.º 6
0
 def test_from_file_raises_with_missing_inputs(self, fasta_filename,
                                               fai_filename):
   fasta = test_utils.genomics_core_testdata(fasta_filename)
   fai = test_utils.genomics_core_testdata(fai_filename)
   with self.assertRaisesRegexp(
       ValueError,
       'Not found: could not load fasta and/or fai for fasta ' + fasta):
     indexed_fasta_reader.IndexedFastaReader.from_file(fasta, fai)
Ejemplo n.º 7
0
    def test_main(self):
        in_ref = test_utils.genomics_core_testdata('test.fasta')
        in_vcf = test_utils.genomics_core_testdata('test_phaseset.vcf')

        with mock.patch.object(sys, 'exit') as mock_exit:
            validate_vcf.main(['validate_vcf', in_ref, in_vcf])
            # Only the first call to sys.exit() matters.
            self.assertEqual(mock.call(-1), mock_exit.mock_calls[0])
Ejemplo n.º 8
0
 def test_iterate(self, fasta_filename):
     # Check the indexed fasta file's iterable matches that of the unindexed
     # fasta file.
     indexed_fasta_reader = fasta.IndexedFastaReader(
         test_utils.genomics_core_testdata(fasta_filename))
     unindexed_fasta_reader = fasta.UnindexedFastaReader(
         test_utils.genomics_core_testdata(fasta_filename))
     self.assertEqual(list(indexed_fasta_reader.iterate()),
                      list(unindexed_fasta_reader.iterate()))
Ejemplo n.º 9
0
 def test_from_file_raises_with_missing_inputs(self, fasta_filename,
                                               fai_filename):
     fasta = test_utils.genomics_core_testdata(fasta_filename)
     fai = test_utils.genomics_core_testdata(fai_filename)
     with self.assertRaisesRegexp(
             ValueError,
             'Not found: could not load fasta and/or fai for fasta ' +
             fasta):
         reference_fai.GenomeReferenceFai.from_file(fasta, fai)
Ejemplo n.º 10
0
 def setUp(self):
   self.unindexed_options = variants_pb2.VcfReaderOptions()
   self.indexed_options = variants_pb2.VcfReaderOptions(
       index_mode=index_pb2.INDEX_BASED_ON_FILENAME)
   self.sites_vcf = test_utils.genomics_core_testdata('test_sites.vcf')
   self.samples_vcf = test_utils.genomics_core_testdata('test_samples.vcf.gz')
   self.sites_reader = vcf_reader.VcfReader.from_file(self.sites_vcf,
                                                      self.unindexed_options)
   self.samples_reader = vcf_reader.VcfReader.from_file(
       self.samples_vcf, self.indexed_options)
Ejemplo n.º 11
0
 def test_from_file_raises_with_missing_inputs(self, fasta_filename,
                                               fai_filename):
     fasta = test_utils.genomics_core_testdata(fasta_filename)
     fai = test_utils.genomics_core_testdata(fai_filename)
     # TODO(b/196638558): OpError exception not propagated.
     with self.assertRaisesRegexp(
             ValueError,
             'could not load fasta and/or fai for fasta ' + fasta):
         reference.IndexedFastaReader.from_file(
             fasta, fai, fasta_pb2.FastaReaderOptions())
Ejemplo n.º 12
0
 def test_dispatching_reader(self):
   with fasta.FastaReader(
       test_utils.genomics_core_testdata('test.fasta')) as reader:
     # The reader is an instance of IndexedFastaReader which supports query().
     self.assertEqual(reader.query(ranges.make_range('chrM', 1, 6)), 'ATCAC')
   with fasta.FastaReader(
       test_utils.genomics_core_testdata('unindexed.fasta')) as reader:
     # The reader is an instance of UnindexedFastaReader which doesn't support
     # query().
     with self.assertRaises(NotImplementedError):
       reader.query(ranges.make_range('chrM', 1, 5))
Ejemplo n.º 13
0
 def _make_reader(self, filename, has_embedded_ref):
     if has_embedded_ref:
         # If we have an embedded reference, force the reader to use it by not
         # providing an argument for ref_path.
         return sam.SamReader(test_utils.genomics_core_testdata(filename))
     else:
         # Otherwise we need to explicitly override the reference encoded in the UR
         # of the CRAM file to use the path provided to our test.fasta.
         return sam.SamReader(
             test_utils.genomics_core_testdata(filename),
             ref_path=test_utils.genomics_core_testdata('test.fasta'))
Ejemplo n.º 14
0
 def setUp(self):
     tfrecord_file = test_utils.genomics_core_testdata(
         'test_features.gff.tfrecord')
     self.records = list(
         io_utils.read_tfrecords(tfrecord_file, proto=gff_pb2.GffRecord))
     self.header = gff_pb2.GffHeader(
         sequence_regions=[ranges.make_range('ctg123', 0, 1497228)])
Ejemplo n.º 15
0
    def test_roundtrip(self,
                       expected_infos,
                       expected_fmt,
                       expected_fmt1,
                       expected_fmt2,
                       reader_excluded_info=None,
                       reader_excluded_format=None,
                       writer_excluded_info=None,
                       writer_excluded_format=None):
        expected_records = [
            record.format(info=info, fmt=expected_fmt, efmts1=e1,
                          efmts2=e2) for record, info, e1, e2 in zip(
                              self.record_format_strings, expected_infos,
                              expected_fmt1, expected_fmt2)
        ]
        expected = self.header + ''.join(expected_records)
        with vcf.VcfReader(
                test_utils.genomics_core_testdata('test_py_roundtrip.vcf'),
                excluded_info_fields=reader_excluded_info,
                excluded_format_fields=reader_excluded_format) as reader:

            records = list(reader.iterate())
            output_path = test_utils.test_tmpfile('test_roundtrip_tmpfile.vcf')
            with vcf.VcfWriter(
                    output_path,
                    header=reader.header,
                    excluded_info_fields=writer_excluded_info,
                    excluded_format_fields=writer_excluded_format) as writer:
                for record in records:
                    writer.write(record)

        with open(output_path) as f:
            actual = f.read()
        self.assertEqual(actual, expected)
Ejemplo n.º 16
0
  def test_c_reader(self):
    self.assertNotEqual(self.sites_reader.c_reader, 0)
    self.assertNotEqual(self.samples_reader.c_reader, 0)

    tfrecord_reader = vcf.VcfReader(
        test_utils.genomics_core_testdata('test_samples.vcf.golden.tfrecord'))
    self.assertNotEqual(tfrecord_reader.c_reader, 0)
Ejemplo n.º 17
0
 def setUp(self):
     super(TabixTest, self).setUp()
     self.input_file = test_utils.genomics_core_testdata(
         'test_samples.vcf.gz')
     self.output_file = test_utils.test_tmpfile('test_samples.vcf.gz')
     shutil.copyfile(self.input_file, self.output_file)
     self.tbx_index_file = self.output_file + '.tbi'
Ejemplo n.º 18
0
 def test_sam_iterate_raises_on_malformed_record(self):
     malformed = test_utils.genomics_core_testdata('malformed.sam')
     reader = sam_reader.SamReader.from_file(malformed, self.options)
     iterable = iter(reader.iterate())
     self.assertIsNotNone(next(iterable))
     with self.assertRaises(ValueError):
         list(iterable)
Ejemplo n.º 19
0
    def test_conversion_to_tfrecord_and_back(self, original_input_file):
        """Test conversion from a native file format to tfrecord.gz, then back."""
        input_path = test_utils.genomics_core_testdata(original_input_file)
        tfrecord_output_path = test_utils.test_tmpfile(original_input_file +
                                                       ".tfrecord.gz")
        native_output_path = test_utils.test_tmpfile(original_input_file)

        # Test conversion from native format to tfrecord.
        self._convert(input_path, tfrecord_output_path)

        # TODO(b/63133103): remove this when SAM writer is implemented.
        if native_output_path.endswith(".sam"):
            raise unittest.SkipTest("SAM writing not yet supported")

        # Test conversion from tfrecord format back to native format.  Ensure that
        # conversions where we would need a header, but don't have one from the
        # input, trigger an error message.
        if any(
                native_output_path.endswith(ext)
                for ext in FORMATS_REQUIRING_HEADER):
            with self.assertRaisesRegexp(
                    converter.ConversionError,
                    "Input file does not have a header, which is needed to construct "
                    "output file"):
                self._convert(tfrecord_output_path, native_output_path)

        else:
            self._convert(tfrecord_output_path, native_output_path)
Ejemplo n.º 20
0
 def test_native_gff_header(self, gff_filename):
     gff_path = test_utils.genomics_core_testdata(gff_filename)
     with gff.GffReader(gff_path) as reader:
         self.assertEqual(EXPECTED_GFF_VERSION, reader.header.gff_version)
     with gff.NativeGffReader(gff_path) as native_reader:
         self.assertEqual(EXPECTED_GFF_VERSION,
                          native_reader.header.gff_version)
Ejemplo n.º 21
0
 def test_headless_sam_raises(self):
     headerless = test_utils.genomics_core_testdata('headerless.sam')
     with self.assertRaisesRegex(
             ValueError, 'Could not parse file with bad SAM header'):
         sam_reader.SamReader.from_file(reads_path=headerless,
                                        ref_path='',
                                        options=self.options)
Ejemplo n.º 22
0
 def test_bed_iterate_raises_on_malformed_record(self, filename):
     malformed = test_utils.genomics_core_testdata(filename)
     reader = bed_reader.BedReader.from_file(malformed, self.options)
     iterable = iter(reader.iterate())
     self.assertIsNotNone(next(iterable))
     with self.assertRaises(ValueError):
         list(iterable)
Ejemplo n.º 23
0
  def test_round_trip_vcf(self, test_datum_name):
    # Round-trip variants through writing and reading:
    # 1. Read variants v1 from VcfReader;
    # 2. Write v1 to vcf using our VcfWriter;
    # 3. Read back in using VcfReader -- v2;
    # 4. compare v1 and v2.
    in_file = test_utils.genomics_core_testdata(test_datum_name)
    out_file = test_utils.test_tmpfile('output_' + test_datum_name)

    v1_reader = vcf.VcfReader(in_file)
    v1_records = list(v1_reader.iterate())
    self.assertTrue(v1_records, 'Reader failed to find records')

    header = copy.deepcopy(v1_reader.header)
    writer_options = variants_pb2.VcfWriterOptions()

    with vcf_writer.VcfWriter.to_file(out_file, header,
                                      writer_options) as writer:
      for record in v1_records:
        writer.write(record)

    v2_reader = vcf.VcfReader(out_file)
    v2_records = list(v2_reader.iterate())

    self.assertEqual(v1_records, v2_records,
                     'Round-tripped variants not as expected')
Ejemplo n.º 24
0
 def test_headless_sam_raises(self):
     headerless = test_utils.genomics_core_testdata('headerless.sam')
     reader = sam_reader.SamReader.from_file(reads_path=headerless,
                                             ref_path='',
                                             options=self.options)
     iterable = iter(reader.iterate())
     with self.assertRaises(ValueError):
         next(iterable)
Ejemplo n.º 25
0
 def setUp(self):
     self.bed = test_utils.genomics_core_testdata('test_regions.bed')
     self.zipped_bed = test_utils.genomics_core_testdata(
         'test_regions.bed.gz')
     self.options = bed_pb2.BedReaderOptions()
     self.first = bed_pb2.BedRecord(reference_name='chr1',
                                    start=10,
                                    end=20,
                                    name='first',
                                    score=100,
                                    strand=bed_pb2.BedRecord.FORWARD_STRAND,
                                    thick_start=12,
                                    thick_end=18,
                                    item_rgb='255,124,1',
                                    block_count=3,
                                    block_sizes='2,6,2',
                                    block_starts='10,12,18')
Ejemplo n.º 26
0
    def test_from_regions(self, regions, expected):
        # For convenience we allow 'test.bed' in our regions but the actual file
        # path is in our testdata directory.
        for i in range(len(regions)):
            if regions[i] == 'test.bed':
                regions[i] = test_utils.genomics_core_testdata('test.bed')

        self.assertEqual(list(ranges.from_regions(regions)), expected)
Ejemplo n.º 27
0
 def test_from_bed(self, bed_filename):
     source = test_utils.genomics_core_testdata(bed_filename)
     self.assertCountEqual([
         ranges.make_range('chr1', 1, 10),
         ranges.make_range('chr2', 20, 30),
         ranges.make_range('chr2', 40, 60),
         ranges.make_range('chr3', 80, 90),
     ], ranges.RangeSet.from_bed(source))
Ejemplo n.º 28
0
 def testCompressedExplicit(self):
     reader = genomics_reader.TFRecordReader(
         test_utils.genomics_core_testdata('test_features.gff.tfrecord.gz'),
         gff_pb2.GffRecord(),
         compression_type='GZIP')
     records = list(reader.iterate())
     self.assertEqual('GenBank', records[0].source)
     self.assertEqual('ctg123', records[1].range.reference_name)
Ejemplo n.º 29
0
 def test_query_on_unindexed_reader_raises(self):
     window = ranges.parse_literal('chr1:10,000,000-10,000,100')
     unindexed_file = test_utils.genomics_core_testdata('test_samples.vcf')
     with vcf_reader.VcfReader.from_file(unindexed_file,
                                         self.options) as reader:
         with self.assertRaisesRegexp(ValueError,
                                      'Cannot query without an index'):
             reader.query(window)
Ejemplo n.º 30
0
 def test_iterate_bed_reader(self, bed_filename):
     bed_path = test_utils.genomics_core_testdata(bed_filename)
     expected = [('chr1', 10, 20), ('chr1', 100, 200)]
     with bed.BedReader(bed_path) as reader:
         records = list(reader.iterate())
     self.assertLen(records, 2)
     self.assertEqual([(r.reference_name, r.start, r.end) for r in records],
                      expected)