def test_generate_tfrecords(self): examples_out = os.path.join(absltest.get_default_test_tmpdir(), 'examples_output') train_test_val_split = [0.7, 0.2, 0.1] ngs_read_length = ngs_errors.generate_tfrecord_datasets( train_test_val_split, ref_path=test_utils.genomics_core_testdata( 'ucsc.hg19.chr20.unittest.fasta.gz'), vcf_path=test_utils.genomics_core_testdata( 'test_nist.b37_chr20_100kbp_at_10mb.vcf.gz'), bam_path=test_utils.genomics_core_testdata( 'NA12878_S1.chr20.10_10p1mb.bam'), out_dir=examples_out, max_reads=100) actual_examples = self._read_examples(train_test_val_split, examples_out) golden_examples = self._read_examples( train_test_val_split, test_utils.genomics_core_testdata('golden.examples.ngs_errors')) self.assertEqual(len(actual_examples), len(golden_examples)) matched_examples = [] for expected in golden_examples: for actual in actual_examples: if all(actual.features.feature[key] == expected.features.feature[key] for key in expected.features.feature.keys()): matched_examples.append(expected) self.assertEqual(golden_examples, matched_examples)
def test_wrap(self, fasta_filename): chr_names = ['chrM', 'chr1', 'chr2'] chr_lengths = [100, 76, 121] fasta = test_utils.genomics_core_testdata(fasta_filename) fai = test_utils.genomics_core_testdata(fasta_filename + '.fai') with reference_fai.GenomeReferenceFai.from_file(fasta, fai) as ref: self.assertEqual(ref.n_contigs, 3) self.assertIn(fasta, ref.fasta_path) self.assertIn('GenomeReference backed by htslib FAI index', str(ref)) self.assertEqual(ref.contig_names, chr_names) self.assertEqual(ref.n_bp, sum(chr_lengths)) self.assertEqual(ref.bases(ranges.make_range('chrM', 1, 10)), 'ATCACAGGT') self.assertTrue( ref.is_valid_interval(ranges.make_range('chrM', 1, 10))) self.assertFalse( ref.is_valid_interval(ranges.make_range('chrM', 1, 100000))) self.assertEqual(len(ref.contigs), 3) self.assertEqual([c.name for c in ref.contigs], chr_names) self.assertEqual([c.n_bases for c in ref.contigs], chr_lengths) for contig in ref.contigs: self.assertEqual(ref.contig(contig.name), contig) self.assertTrue(ref.has_contig(contig.name)) self.assertFalse(ref.has_contig(contig.name + '.unknown'))
def test_main(self): examples_out = test_utils.test_tmpfile('output.tfrecord') ngs_errors.make_ngs_error_examples( ref_path=test_utils.genomics_core_testdata( 'ucsc.hg19.chr20.unittest.fasta.gz'), vcf_path=test_utils.genomics_core_testdata( 'test_nist.b37_chr20_100kbp_at_10mb.vcf.gz'), bam_path=test_utils.genomics_core_testdata( 'NA12878_S1.chr20.10_10p1mb.bam'), examples_out_path=examples_out, max_reads=100) actual_examples = _read_examples(examples_out) golden_examples = _read_examples( test_utils.genomics_core_testdata( 'golden.examples.ngs_errors.tfrecord')) self.assertEqual(len(actual_examples), 100) assertExamplesAreEqual(self, golden_examples, actual_examples, expected_keys={ 'read_name', 'cigar', 'read_sequence', 'read_qualities', 'true_sequence' })
def setUp(self): self.sites_vcf = test_utils.genomics_core_testdata('test_sites.vcf') self.samples_vcf = test_utils.genomics_core_testdata('test_samples.vcf.gz') self.options = variants_pb2.VcfReaderOptions() self.sites_reader = vcf_reader.VcfReader.from_file(self.sites_vcf, self.options) self.samples_reader = vcf_reader.VcfReader.from_file( self.samples_vcf, self.options)
def setUp(self): self.sites_reader = vcf.VcfReader( test_utils.genomics_core_testdata('test_sites.vcf'), use_index=False) self.samples_reader = vcf.VcfReader( test_utils.genomics_core_testdata('test_samples.vcf.gz'), use_index=True)
def test_from_file_raises_with_missing_inputs(self, fasta_filename, fai_filename): fasta = test_utils.genomics_core_testdata(fasta_filename) fai = test_utils.genomics_core_testdata(fai_filename) with self.assertRaisesRegexp( ValueError, 'Not found: could not load fasta and/or fai for fasta ' + fasta): indexed_fasta_reader.IndexedFastaReader.from_file(fasta, fai)
def test_main(self): in_ref = test_utils.genomics_core_testdata('test.fasta') in_vcf = test_utils.genomics_core_testdata('test_phaseset.vcf') with mock.patch.object(sys, 'exit') as mock_exit: validate_vcf.main(['validate_vcf', in_ref, in_vcf]) # Only the first call to sys.exit() matters. self.assertEqual(mock.call(-1), mock_exit.mock_calls[0])
def test_iterate(self, fasta_filename): # Check the indexed fasta file's iterable matches that of the unindexed # fasta file. indexed_fasta_reader = fasta.IndexedFastaReader( test_utils.genomics_core_testdata(fasta_filename)) unindexed_fasta_reader = fasta.UnindexedFastaReader( test_utils.genomics_core_testdata(fasta_filename)) self.assertEqual(list(indexed_fasta_reader.iterate()), list(unindexed_fasta_reader.iterate()))
def test_from_file_raises_with_missing_inputs(self, fasta_filename, fai_filename): fasta = test_utils.genomics_core_testdata(fasta_filename) fai = test_utils.genomics_core_testdata(fai_filename) with self.assertRaisesRegexp( ValueError, 'Not found: could not load fasta and/or fai for fasta ' + fasta): reference_fai.GenomeReferenceFai.from_file(fasta, fai)
def setUp(self): self.unindexed_options = variants_pb2.VcfReaderOptions() self.indexed_options = variants_pb2.VcfReaderOptions( index_mode=index_pb2.INDEX_BASED_ON_FILENAME) self.sites_vcf = test_utils.genomics_core_testdata('test_sites.vcf') self.samples_vcf = test_utils.genomics_core_testdata('test_samples.vcf.gz') self.sites_reader = vcf_reader.VcfReader.from_file(self.sites_vcf, self.unindexed_options) self.samples_reader = vcf_reader.VcfReader.from_file( self.samples_vcf, self.indexed_options)
def test_from_file_raises_with_missing_inputs(self, fasta_filename, fai_filename): fasta = test_utils.genomics_core_testdata(fasta_filename) fai = test_utils.genomics_core_testdata(fai_filename) # TODO(b/196638558): OpError exception not propagated. with self.assertRaisesRegexp( ValueError, 'could not load fasta and/or fai for fasta ' + fasta): reference.IndexedFastaReader.from_file( fasta, fai, fasta_pb2.FastaReaderOptions())
def test_dispatching_reader(self): with fasta.FastaReader( test_utils.genomics_core_testdata('test.fasta')) as reader: # The reader is an instance of IndexedFastaReader which supports query(). self.assertEqual(reader.query(ranges.make_range('chrM', 1, 6)), 'ATCAC') with fasta.FastaReader( test_utils.genomics_core_testdata('unindexed.fasta')) as reader: # The reader is an instance of UnindexedFastaReader which doesn't support # query(). with self.assertRaises(NotImplementedError): reader.query(ranges.make_range('chrM', 1, 5))
def _make_reader(self, filename, has_embedded_ref): if has_embedded_ref: # If we have an embedded reference, force the reader to use it by not # providing an argument for ref_path. return sam.SamReader(test_utils.genomics_core_testdata(filename)) else: # Otherwise we need to explicitly override the reference encoded in the UR # of the CRAM file to use the path provided to our test.fasta. return sam.SamReader( test_utils.genomics_core_testdata(filename), ref_path=test_utils.genomics_core_testdata('test.fasta'))
def setUp(self): tfrecord_file = test_utils.genomics_core_testdata( 'test_features.gff.tfrecord') self.records = list( io_utils.read_tfrecords(tfrecord_file, proto=gff_pb2.GffRecord)) self.header = gff_pb2.GffHeader( sequence_regions=[ranges.make_range('ctg123', 0, 1497228)])
def test_roundtrip(self, expected_infos, expected_fmt, expected_fmt1, expected_fmt2, reader_excluded_info=None, reader_excluded_format=None, writer_excluded_info=None, writer_excluded_format=None): expected_records = [ record.format(info=info, fmt=expected_fmt, efmts1=e1, efmts2=e2) for record, info, e1, e2 in zip( self.record_format_strings, expected_infos, expected_fmt1, expected_fmt2) ] expected = self.header + ''.join(expected_records) with vcf.VcfReader( test_utils.genomics_core_testdata('test_py_roundtrip.vcf'), excluded_info_fields=reader_excluded_info, excluded_format_fields=reader_excluded_format) as reader: records = list(reader.iterate()) output_path = test_utils.test_tmpfile('test_roundtrip_tmpfile.vcf') with vcf.VcfWriter( output_path, header=reader.header, excluded_info_fields=writer_excluded_info, excluded_format_fields=writer_excluded_format) as writer: for record in records: writer.write(record) with open(output_path) as f: actual = f.read() self.assertEqual(actual, expected)
def test_c_reader(self): self.assertNotEqual(self.sites_reader.c_reader, 0) self.assertNotEqual(self.samples_reader.c_reader, 0) tfrecord_reader = vcf.VcfReader( test_utils.genomics_core_testdata('test_samples.vcf.golden.tfrecord')) self.assertNotEqual(tfrecord_reader.c_reader, 0)
def setUp(self): super(TabixTest, self).setUp() self.input_file = test_utils.genomics_core_testdata( 'test_samples.vcf.gz') self.output_file = test_utils.test_tmpfile('test_samples.vcf.gz') shutil.copyfile(self.input_file, self.output_file) self.tbx_index_file = self.output_file + '.tbi'
def test_sam_iterate_raises_on_malformed_record(self): malformed = test_utils.genomics_core_testdata('malformed.sam') reader = sam_reader.SamReader.from_file(malformed, self.options) iterable = iter(reader.iterate()) self.assertIsNotNone(next(iterable)) with self.assertRaises(ValueError): list(iterable)
def test_conversion_to_tfrecord_and_back(self, original_input_file): """Test conversion from a native file format to tfrecord.gz, then back.""" input_path = test_utils.genomics_core_testdata(original_input_file) tfrecord_output_path = test_utils.test_tmpfile(original_input_file + ".tfrecord.gz") native_output_path = test_utils.test_tmpfile(original_input_file) # Test conversion from native format to tfrecord. self._convert(input_path, tfrecord_output_path) # TODO(b/63133103): remove this when SAM writer is implemented. if native_output_path.endswith(".sam"): raise unittest.SkipTest("SAM writing not yet supported") # Test conversion from tfrecord format back to native format. Ensure that # conversions where we would need a header, but don't have one from the # input, trigger an error message. if any( native_output_path.endswith(ext) for ext in FORMATS_REQUIRING_HEADER): with self.assertRaisesRegexp( converter.ConversionError, "Input file does not have a header, which is needed to construct " "output file"): self._convert(tfrecord_output_path, native_output_path) else: self._convert(tfrecord_output_path, native_output_path)
def test_native_gff_header(self, gff_filename): gff_path = test_utils.genomics_core_testdata(gff_filename) with gff.GffReader(gff_path) as reader: self.assertEqual(EXPECTED_GFF_VERSION, reader.header.gff_version) with gff.NativeGffReader(gff_path) as native_reader: self.assertEqual(EXPECTED_GFF_VERSION, native_reader.header.gff_version)
def test_headless_sam_raises(self): headerless = test_utils.genomics_core_testdata('headerless.sam') with self.assertRaisesRegex( ValueError, 'Could not parse file with bad SAM header'): sam_reader.SamReader.from_file(reads_path=headerless, ref_path='', options=self.options)
def test_bed_iterate_raises_on_malformed_record(self, filename): malformed = test_utils.genomics_core_testdata(filename) reader = bed_reader.BedReader.from_file(malformed, self.options) iterable = iter(reader.iterate()) self.assertIsNotNone(next(iterable)) with self.assertRaises(ValueError): list(iterable)
def test_round_trip_vcf(self, test_datum_name): # Round-trip variants through writing and reading: # 1. Read variants v1 from VcfReader; # 2. Write v1 to vcf using our VcfWriter; # 3. Read back in using VcfReader -- v2; # 4. compare v1 and v2. in_file = test_utils.genomics_core_testdata(test_datum_name) out_file = test_utils.test_tmpfile('output_' + test_datum_name) v1_reader = vcf.VcfReader(in_file) v1_records = list(v1_reader.iterate()) self.assertTrue(v1_records, 'Reader failed to find records') header = copy.deepcopy(v1_reader.header) writer_options = variants_pb2.VcfWriterOptions() with vcf_writer.VcfWriter.to_file(out_file, header, writer_options) as writer: for record in v1_records: writer.write(record) v2_reader = vcf.VcfReader(out_file) v2_records = list(v2_reader.iterate()) self.assertEqual(v1_records, v2_records, 'Round-tripped variants not as expected')
def test_headless_sam_raises(self): headerless = test_utils.genomics_core_testdata('headerless.sam') reader = sam_reader.SamReader.from_file(reads_path=headerless, ref_path='', options=self.options) iterable = iter(reader.iterate()) with self.assertRaises(ValueError): next(iterable)
def setUp(self): self.bed = test_utils.genomics_core_testdata('test_regions.bed') self.zipped_bed = test_utils.genomics_core_testdata( 'test_regions.bed.gz') self.options = bed_pb2.BedReaderOptions() self.first = bed_pb2.BedRecord(reference_name='chr1', start=10, end=20, name='first', score=100, strand=bed_pb2.BedRecord.FORWARD_STRAND, thick_start=12, thick_end=18, item_rgb='255,124,1', block_count=3, block_sizes='2,6,2', block_starts='10,12,18')
def test_from_regions(self, regions, expected): # For convenience we allow 'test.bed' in our regions but the actual file # path is in our testdata directory. for i in range(len(regions)): if regions[i] == 'test.bed': regions[i] = test_utils.genomics_core_testdata('test.bed') self.assertEqual(list(ranges.from_regions(regions)), expected)
def test_from_bed(self, bed_filename): source = test_utils.genomics_core_testdata(bed_filename) self.assertCountEqual([ ranges.make_range('chr1', 1, 10), ranges.make_range('chr2', 20, 30), ranges.make_range('chr2', 40, 60), ranges.make_range('chr3', 80, 90), ], ranges.RangeSet.from_bed(source))
def testCompressedExplicit(self): reader = genomics_reader.TFRecordReader( test_utils.genomics_core_testdata('test_features.gff.tfrecord.gz'), gff_pb2.GffRecord(), compression_type='GZIP') records = list(reader.iterate()) self.assertEqual('GenBank', records[0].source) self.assertEqual('ctg123', records[1].range.reference_name)
def test_query_on_unindexed_reader_raises(self): window = ranges.parse_literal('chr1:10,000,000-10,000,100') unindexed_file = test_utils.genomics_core_testdata('test_samples.vcf') with vcf_reader.VcfReader.from_file(unindexed_file, self.options) as reader: with self.assertRaisesRegexp(ValueError, 'Cannot query without an index'): reader.query(window)
def test_iterate_bed_reader(self, bed_filename): bed_path = test_utils.genomics_core_testdata(bed_filename) expected = [('chr1', 10, 20), ('chr1', 100, 200)] with bed.BedReader(bed_path) as reader: records = list(reader.iterate()) self.assertLen(records, 2) self.assertEqual([(r.reference_name, r.start, r.end) for r in records], expected)