def test_realigner_end2end(self): ref_reader = genomics_io.make_ref_reader(test_utils.CHR20_FASTA) config = realigner.realigner_config(FLAGS) reads_realigner = realigner.Realigner(config, ref_reader) region_str = 'chr20:10,000,000-10,009,999' regions = ranges.RangeSet.from_regions([region_str]) for region in regions.partition(1000): with genomics_io.make_sam_reader( test_utils.CHR20_BAM, core_pb2.ReadRequirements()) as sam_reader: in_reads = list(sam_reader.query(region)) windows, out_reads = reads_realigner.realign_reads( in_reads, region) # We should always get back all of the reads we sent in. Instead of just # checking the lengths are the same, make sure all the read names are the # same. self.assertCountEqual([r.fragment_name for r in in_reads], [r.fragment_name for r in out_reads]) # Make sure we assembled at least one windows in the region. self.assertNotEqual(0, len(windows)) # Check each window to make sure it's reasonable. for window in windows: # We always expect the reference sequence to be one of our haplotypes. ref_seq = ref_reader.bases(window.span) self.assertIn(ref_seq, set(window.haplotypes))
def _make_sam_reader(self): return genomics_io.make_sam_reader( self.options.reads_filename, self.options.read_requirements, hts_block_size=FLAGS.hts_block_size, downsample_fraction=self.options.downsample_fraction, random_seed=self.options.random_seed)
def test_call_from_allele_counter(self): ref = genomics_io.make_ref_reader(test_utils.CHR20_FASTA) sam_reader = genomics_io.make_sam_reader(test_utils.CHR20_BAM) size = 1000 region = ranges.make_range('chr20', 10000000, 10000000 + size) allele_counter = _allelecounter.AlleleCounter( ref, region, deepvariant_pb2.AlleleCounterOptions(partition_size=size)) caller = variant_calling.VariantCaller( deepvariant_pb2.VariantCallerOptions(min_count_snps=2, min_count_indels=2, min_fraction_snps=0.12, min_fraction_indels=0.12, sample_name='sample_name', p_error=0.001, max_gq=50, gq_resolution=1, ploidy=2)) # Grab all of the reads in our region and add them to the allele_counter. reads = list(sam_reader.query(region)) self.assertNotEmpty(reads) for read in reads: allele_counter.add(read) # Get the candidates records for this whole region. candidates = caller.calls_from_allele_counter(allele_counter) # We should have at least some candidates and some gvcf records. self.assertNotEmpty(candidates) # Each candidate should be a DeepVariantCall. for candidate in candidates: self.assertIsInstance(candidate, deepvariant_pb2.DeepVariantCall)
def test_sam_query(self): reader = genomics_io.make_sam_reader( test_utils.genomics_core_testdata('test.bam')) expected = [(ranges.parse_literal('chr20:10,000,000-10,000,100'), 106), (ranges.parse_literal('chr20:10,000,000-10,000,000'), 45)] with reader: for interval, n_expected in expected: with reader.query(interval) as iterable: self.assertEqual(test_utils.iterable_len(iterable), n_expected)
def test_straightforward_region(self): ref_reader = genomics_io.make_ref_reader(test_utils.CHR20_FASTA) bam_reader = genomics_io.make_sam_reader(test_utils.CHR20_BAM) region = ranges.parse_literal('chr20:10,000,000-10,000,100') ref_seq = ref_reader.bases(region) all_reads = list(bam_reader.query(region)) dbg30 = debruijn_graph.build(ref_seq, all_reads, self.single_k_dbg_options(30)) self.assertIsNotNone(dbg30) self.assertEqual([ref_seq], dbg30.candidate_haplotypes())
def test_bam_iterate_partially(self): """Verify that iteration provides results incrementally, not all at once.""" reader = genomics_io.make_sam_reader( test_utils.genomics_core_testdata('test.bam'), use_index=False) with reader: iterable = reader.iterate() # We expect 106 records in total. for _ in xrange(10): results = list(itertools.islice(iterable, 10)) self.assertEqual(len(results), 10) results = list(itertools.islice(iterable, 10)) self.assertEqual(len(results), 6)
def _parse_read_with_aux_tags(self, tag_string): # Minimal header line to create a valid SAM file. header_lines = '@HD VN:1.3 SO:coordinate\n@SQ SN:chr1 LN:248956422\n' # A single stock read we'll add our AUX fields to. read = 'read_name 0 chr1 1 0 3M * 0 0 CCC AAA ' + tag_string path = test_utils.test_tmpfile('aux_tags.bam') with tf.gfile.FastGFile(path, 'w') as fout: fout.write(header_lines) fout.write(read + '\n') with genomics_io.make_sam_reader( path, use_index=False, parse_aux_fields=True) as reader: return list(reader.iterate())
def test_downsampling(self, method, maybe_range, fraction, expected_n_reads): reader = genomics_io.make_sam_reader( test_utils.genomics_core_testdata('test.bam'), downsample_fraction=fraction, random_seed=12345) with reader: if method == 'iterate': reads_iter = reader.iterate() elif method == 'query': reads_iter = reader.query(ranges.parse_literal(maybe_range)) else: self.fail('Unexpected method', method) self.assertEqual(test_utils.iterable_len(reads_iter), expected_n_reads)
def test_complex_region(self): # There is a heterozygous 9 bp deletion of tandem TGA repeat. # "chr20:10,095,379-10,095,500" ref_reader = genomics_io.make_ref_reader(test_utils.CHR20_FASTA) bam_reader = genomics_io.make_sam_reader(test_utils.CHR20_BAM) region = ranges.parse_literal('chr20:10,095,379-10,095,500') ref_seq = ref_reader.bases(region) reads = list(bam_reader.query(region)) dbg = debruijn_graph.build(ref_seq, reads, self.dbg_options()) self.assertIsNotNone(dbg) self.assertEqual(44, dbg.kmer_size) self.assertEqual(2, len(dbg.candidate_haplotypes())) self.assertIn(ref_seq, dbg.candidate_haplotypes())
def processing_regions_from_options(options): """Computes the calling regions from our options. This function does all of the work needed to read our input files and region specifications to determine the list of regions we should generate examples over. It also computes the confident regions need to label variants. Args: options: deepvariant.DeepVariantOptions proto containing information about our input data sources. Returns: Two values. The first is a list of learning.genomics.v1.Range protos of the regions we should process. The second is a RangeSet containing the confident regions for labeling, or None if we are running in training mode. """ ref_contigs = genomics_io.make_ref_reader( options.reference_filename).contigs sam_contigs = genomics_io.make_sam_reader(options.reads_filename).contigs # Add in confident regions and vcf_contigs if in training mode. vcf_contigs = None if in_training_mode(options): vcf_contigs = genomics_io.make_vcf_reader( options.truth_variants_filename).contigs # Compute the common contigs among our inputs, and check that the contigs are # sufficiently consistent among each other. contigs = common_contigs(only_true(ref_contigs, sam_contigs, vcf_contigs), exclude_contig_names=options.exclude_contigs) validate_reference_contig_coverage(ref_contigs, contigs, options.min_shared_contigs_basepairs) logging.info('Common contigs are %s', [c.name for c in contigs]) regions = regions_to_process( contigs, partition_size=options.allele_counter_options.partition_size, calling_regions=ranges.RangeSet.from_regions( options.calling_regions, ranges.contigs_dict(ref_contigs)), task_id=options.task_id, num_shards=options.num_shards) return regions
def extract_sample_name_from_reads(reads_path): """Returns the sample name as derived from the BAM file of reads. Args: reads_path: Path to the SAM/BAM file containing a single sample. Returns: The sample ID annotated in the read group. Raises: ValueError: There is not exactly one unique sample name in the SAM/BAM. """ with genomics_io.make_sam_reader(reads_path) as sam_reader: samples = sam_reader.samples if len(samples) != 1: raise ValueError('Expected a single sample, found {}'.format(samples)) sample = next(iter(samples)) if not sample: raise ValueError('Sample name is empty.') return sample
def _get_reads(region): with genomics_io.make_sam_reader(test_utils.CHR20_BAM) as in_sam_reader: return list(in_sam_reader.query(region))
def test_tfbam_plugin_loads(self): reader = genomics_io.make_sam_reader('*****@*****.**', use_index=True) self.assertIsNotNone(reader)
def test_bam_iterate(self): reader = genomics_io.make_sam_reader( test_utils.genomics_core_testdata('test.bam'), use_index=False) with reader: self.assertEqual(test_utils.iterable_len(reader.iterate()), 106)
def test_tfbam_plugin_does_not_load(self): with self.assertRaisesRegexp( ImportError, 'tfbam_lib module not found, cannot read .tfbam files.'): _ = genomics_io.make_sam_reader('*****@*****.**', use_index=True)
def test_remote_bam(self): reader = genomics_io.make_sam_reader(REMOTE_BAM) reads = list(reader.query(self.query_window)) self.assertEqual(EXPECTED_READS_IN_WINDOW, len(reads))
def default_options(add_flags=True, flags_obj=None): """Creates a DeepVariantOptions proto populated with reasonable defaults. Args: add_flags: bool. defaults to True. If True, we will push the value of certain FLAGS into our options. If False, those option fields are left uninitialized. flags_obj: object. If not None, use as the source of flags, else use global FLAGS. Returns: deepvariant_pb2.DeepVariantOptions protobuf. Raises: ValueError: If we observe invalid flag values. """ if not flags_obj: flags_obj = FLAGS read_reqs = core_pb2.ReadRequirements( min_base_quality=10, min_mapping_quality=10, min_base_quality_mode=core_pb2.ReadRequirements.ENFORCED_BY_CLIENT) pic_options = pileup_image.default_options(read_requirements=read_reqs) allele_counter_options = deepvariant_pb2.AlleleCounterOptions( partition_size=flags_obj.partition_size, read_requirements=read_reqs) if flags_obj.sample_name: sample_name = flags_obj.sample_name elif flags_obj.reads: with genomics_io.make_sam_reader(flags_obj.reads) as sam_reader: sample_name = extract_sample_name_from_sam_reader(sam_reader) else: sample_name = _UNKNOWN_SAMPLE variant_caller_options = deepvariant_pb2.VariantCallerOptions( min_count_snps=flags_obj.vsc_min_count_snps, min_count_indels=flags_obj.vsc_min_count_indels, min_fraction_snps=flags_obj.vsc_min_fraction_snps, min_fraction_indels=flags_obj.vsc_min_fraction_indels, # Not specified by default: fraction_reference_sites_to_emit, # Fixed random seed produced with 'od -vAn -N4 -tu4 < /dev/urandom'. random_seed=1400605801, sample_name=sample_name, p_error=0.001, max_gq=50, gq_resolution=flags_obj.gvcf_gq_binsize, ploidy=2) options = deepvariant_pb2.DeepVariantOptions( exclude_contigs=exclude_contigs.EXCLUDED_HUMAN_CONTIGS, # Fixed random seed produced with 'od -vAn -N4 -tu4 < /dev/urandom'. random_seed=609314161, # # Not specified by default: calling_regions = 3; read_requirements=read_reqs, allele_counter_options=allele_counter_options, variant_caller_options=variant_caller_options, pic_options=pic_options, n_cores=1, task_id=0, num_shards=0, min_shared_contigs_basepairs=0.9, ) if add_flags: if flags_obj.mode == 'training': options.mode = deepvariant_pb2.DeepVariantOptions.TRAINING elif flags_obj.mode == 'calling': options.mode = deepvariant_pb2.DeepVariantOptions.CALLING else: raise ValueError('Unexpected mode', flags_obj.mode) if flags_obj.ref: options.reference_filename = flags_obj.ref if flags_obj.reads: options.reads_filename = flags_obj.reads if flags_obj.confident_regions: options.confident_regions_filename = flags_obj.confident_regions if flags_obj.truth_variants: options.truth_variants_filename = flags_obj.truth_variants if flags_obj.downsample_fraction != NO_DOWNSAMPLING: options.downsample_fraction = flags_obj.downsample_fraction if flags_obj.multi_allelic_mode: multi_allelic_enum = { 'include_het_alt_images': deepvariant_pb2.PileupImageOptions.ADD_HET_ALT_IMAGES, 'exclude_het_alt_images': deepvariant_pb2.PileupImageOptions.NO_HET_ALT_IMAGES, }[flags_obj.multi_allelic_mode] options.pic_options.multi_allelic_mode = multi_allelic_enum if flags_obj.pileup_image_height: options.pic_options.height = flags_obj.pileup_image_height if flags_obj.pileup_image_width: options.pic_options.width = flags_obj.pileup_image_width num_shards, examples, candidates, gvcf = io_utils.resolve_filespecs( flags_obj.task, flags_obj.examples or '', flags_obj.candidates or '', flags_obj.gvcf or '') options.examples_filename = examples options.candidates_filename = candidates options.gvcf_filename = gvcf options.calling_regions.extend(parse_regions_flag(flags_obj.regions)) options.exclude_calling_regions.extend( parse_regions_flag(flags_obj.exclude_regions)) options.task_id = flags_obj.task options.num_shards = 0 if num_shards is None else num_shards options.realigner_enabled = flags_obj.realign_reads if options.realigner_enabled: options.realigner_options.CopyFrom( realigner.realigner_config(flags_obj)) options.max_reads_per_partition = flags_obj.max_reads_per_partition if (options.mode == deepvariant_pb2.DeepVariantOptions.TRAINING and flags_obj.training_random_emit_ref_sites != NO_RANDOM_REF): options.variant_caller_options.fraction_reference_sites_to_emit = ( flags_obj.training_random_emit_ref_sites) return options