def test_match_to_gt_haps(self): gt_vcf_in = tk_io.VariantFileReader( os.path.join(TEST_FILE_DIR, 'phasing_gt.vcf.gz')) res = match_to_gt_haps(gt_vcf_in, gt_vcf_in, 'chr1', 0, 31) self.assertEqual(res[0], 0) res = match_to_gt_haps(gt_vcf_in, gt_vcf_in, 'chr1', 500, 531) assert (res is None) vcf_in = tk_io.VariantFileReader( os.path.join(TEST_FILE_DIR, 'phasing.vcf.gz')) res = match_to_gt_haps(gt_vcf_in, vcf_in, 'chr1', 0, 31) self.assertEqual(res[0], 0) res = match_to_gt_haps(gt_vcf_in, vcf_in, 'chr1', 31, 61) self.assertEqual(res[0], 1) res = match_to_gt_haps(gt_vcf_in, vcf_in, 'chr1', 61, 120) assert (res is None)
def main(args, outs): args.coerce_strings() outs.coerce_strings() input_vfr = tk_io.VariantFileReader(args.input) bc_mix_prob = args.bc_mix_prob min_var_hap_conf = args.min_var_hap_conf min_junction_hap_conf = args.min_junction_hap_conf hap_block_size = args.hap_block_size hap_block_buffer_size = args.hap_block_buffer_size max_reassign_rounds = args.max_reassign_rounds chrom, start, stop = tk_io.get_locus_info(args.locus) output_file = open(outs.default.strip('.gz'), 'w') fragment_output_file = open(outs.fragment_phasing.strip('.gz'), 'w') vc_mode, _, _, _ = tk_io.get_vc_mode(args.vc_precalled, args.vc_mode) # Add the component name and the version of the phasing code new_source = "10X/pipelines/stages/snpindels/phase_snpindels %s" % martian.get_pipelines_version( ) new_filters = [ ("10X_PHASING_INCONSISTENT", "Uses haplotype information from the fragments and the alleles to filter some variants that are not consistent with phasing." ), ("10X_HOMOPOLYMER_UNPHASED_INSERTION", "Unphased insertions in homopolymer regions tend to be false positives" ) ] new_formats = [ ("PS", 1, "Integer", "ID of Phase Set for Variant"), ("PQ", 1, "Integer", "Phred QV indicating probability at this variant is incorrectly phased" ), ("JQ", 1, "Integer", "Phred QV indicating probability of a phasing switch error in gap prior to this variant" ), ] vfw = tk_io.VariantFileWriter(output_file, template_file=open(args.input), new_source=new_source, new_format_fields=new_formats, new_filters=new_filters) if args.do_phasing: phaser = Phaser(input_vfr, args.fragments, chrom, start, stop, bc_mix_prob, min_junction_hap_conf, min_var_hap_conf, hap_block_buffer_size, hap_block_size, max_reassign_rounds, vc_mode) phaser.call_haps(vfw, fragment_output_file) else: pass_variants(input_vfr, vfw, chrom, start, stop, strip_phasing_info=True) output_file.close() fragment_output_file.close() tk_tabix.sort_unique_tabix_vcf(outs.default.strip('.gz'))
def join_simple(vcf_left, vcf_right, vcf_out, ff_left, ff_right, ff_out, loc_left, loc_right): frags_left = pandas.read_table(ff_left, header=None, names=frag_col_names) frags_right = pandas.read_table(ff_right, header=None, names=frag_col_names) # make sure chrom is formatted as string categorical frags_left.chrom = frags_left.chrom.astype('str').astype('category') frags_right.chrom = frags_right.chrom.astype('str').astype('category') vfr_left = tk_io.VariantFileReader(vcf_left + ".gz") vfr_right = tk_io.VariantFileReader(vcf_right + ".gz") with open(vcf_left + ".gz", 'r') as template_in, open(vcf_out, 'w') as results_out: vfw = tk_io.VariantFileWriter(results_out, template_file=template_in) stitch(vfr_left, vfr_right, vfw, frags_left, frags_right, ff_out, loc_left, loc_right) tk_tabix.sort_unique_tabix_vcf(vcf_out)
def get_phase_blocks_from_vcf(vcf, filter_trivial=True, locus=None): if locus is not None: (loc_chrom, loc_start, loc_end) = locus else: (loc_chrom, loc_start, loc_end) = (None, None, None) records = tk_io.VariantFileReader(vcf).record_getter(fetch_chrom=loc_chrom, fetch_start=loc_start, fetch_end=loc_end) phase_set, chrom, start, end, length = [], [], [], [], [] curr_set, curr_chrom, curr_start, curr_end = None, None, None, None def get_record_data(record): record_set = tk_io.get_record_phase_set(record) record_chrom = tk_io.get_record_chrom(record) record_pos = tk_io.get_record_pos(record) - 1 return (record_set, record_chrom, record_pos) def write_curr(): phase_set.append(curr_set) chrom.append(curr_chrom) start.append(curr_start) end.append(curr_end) length.append(curr_end - curr_start) for record in records: (next_set, next_chrom, next_pos) = get_record_data(record) if curr_set is None and next_set is not None: # initialize curr_set, curr_chrom, curr_start, curr_end = next_set, next_chrom, next_pos, next_pos elif next_set is not None and (curr_set != next_set or curr_chrom != next_chrom): # new set. write curr, then update all write_curr() curr_set, curr_chrom, curr_start, curr_end = next_set, next_chrom, next_pos, next_pos elif next_set is not None: # continuation of set. just update end curr_end = next_pos # write the last set if curr_set is not None: write_curr() pb = pandas.DataFrame({ 'chrom': chrom, 'phase_set': phase_set, 'start': start, 'end': end, 'length': length }) if filter_trivial: pb = pb[(pb.length > 0) & (pb.phase_set != -1)] return pb[['chrom', 'phase_set', 'start', 'end', 'length']].sort(['chrom', 'phase_set']).reset_index(drop=True)
def vcf_record_iter(in_filename, min_snp_qual): in_vcf = tk_io.VariantFileReader(in_filename) for record in in_vcf.record_getter(restrict_type='snp'): # Only support 1 ALT if len(record.ALT) > 1: continue assert len(record.ALT) == 1 # Filter SNP based on call quality if record.QUAL < min_snp_qual: continue yield record
def sample_by_locus(vcf, locus): (chrom, start, end) = locus recs = tk_io.VariantFileReader(vcf).record_getter(fetch_chrom=chrom, fetch_start=start, fetch_end=end) total_recs = 0 het_snp_recs = 0 het_snp_bcs = 0 for rec in recs: total_recs += 1 if rec.var_type == 'snp' and not tk_io.get_record_homozygous(rec): het_snp_recs += 1 bcs_per_hap = tk_io.get_record_barcodes(rec) het_snp_bcs += sum([len(hap) for hap in bcs_per_hap]) return (total_recs, het_snp_recs, het_snp_bcs)
def main(args, outs): vc_mode, variant_caller, precalled_file, gatk_path = tk_io.get_vc_mode( args.vc_precalled, args.variant_mode) locus = args.locus (chrom, start, stop) = tk_io.get_locus_info(locus) fasta_path = tk_reference.get_fasta(args.reference_path) bedfile = outs.default + ".bed" regions = Regions() if args.targets_file is not None: for (chrom, start, end) in tk_io.get_bed_iterator(args.targets_file, args.locus): regions.add_region((start, end)) else: (chrom, start, stop) = tk_io.get_locus_info(args.locus) regions.add_region((start, stop)) coverage_regions = None if (vc_mode != "precalled") and args.high_coverage_excluded_bed is not None: coverage_regions = get_coverage_regions(args) regions = regions.intersect(coverage_regions) bed_length = 0 with open(bedfile, 'w') as bed_writer: for region in regions.get_region_list(): (start, end) = region bed_writer.write(chrom + "\t" + str(start) + "\t" + str(end) + "\n") bed_length += 1 if vc_mode == "precalled" or vc_mode == "precalled_plus": outs.default = None precalled_vars_path = args.split_input vcf = tk_io.VariantFileReader(precalled_vars_path) with open(outs.precalled, "w") as file_write: output = tk_io.VariantFileWriter( file_write, template_file=open(precalled_vars_path)) variant_iter = tk_io.get_variant_iterator_pos( vcf, bedfile, args.locus) for record in variant_iter: output.write_record(record) if not (vc_mode == "precalled"): outs.precalled = None primary_contigs = tk_reference.load_primary_contigs( args.reference_path) if bed_length > 0 and chrom in primary_contigs: vc.run_variant_caller(variant_caller, gatk_path, args.__mem_gb, fasta_path, args.input, outs.default, bedfile)
def canonicalize(filename, output_name): with open(output_name + "tmp.vcf", 'w') as canon_file: # Left-align indels and output variants as constitutent indels tenkit.log_subprocess.check_call([ 'vcfallelicprimitives', '--keep-info', '-t', 'VCFALLELICPRIMITIVE', filename ], stdout=canon_file) with open(output_name + "tmp2.vcf", 'w') as fixed_vcf: # the reason we are doing this is because vcfallelicprimitives screws up the vcf format in some places in the info fields # it changes some tags from KEY=.; to KEY=; which is invalid. bcftools fixes this, but we dont actually want to filter anything # this should do that tenkit.log_subprocess.check_call( ['bcftools', 'filter', output_name + "tmp.vcf"], stdout=fixed_vcf) with open(output_name + "tmp3.vcf", 'w') as unphased_file: vcf_in = tk_io.VariantFileReader(output_name + "tmp2.vcf") unsupported_genotype_filter = [( "UNSUPPORTED_GENOTYPE", "If genotype field contains '.' we assume that this is due to making a single sample vcf from a multiple sample vcf in which this sample does not contain the variant." )] tenx = ('TENX', '0', 'Flag', "called by 10X", None, None) vcf_out = tk_io.VariantFileWriter( unphased_file, template_file=open(output_name + "tmp2.vcf"), new_info_fields=[tenx], new_filters=unsupported_genotype_filter) for record in vcf_in.record_getter(): sample = record.samples[0] unsupported_genotype = False try: if len(sample.gt_alleles) > 0: genotype1 = sample.gt_alleles[0] if genotype1 == '.': unsupported_genotype = True else: unsupported_genotype = True if len(sample.gt_alleles) > 1: genotype2 = sample.gt_alleles[1] if genotype2 is '.': unsupported_genotype = True except: unsupported_genotype = True if unsupported_genotype: record.FILTER = ["UNSUPPORTED_GENOTYPE"] vcf_out.write_record(record) tk_tabix.sort_vcf(output_name + "tmp3.vcf", output_name)
def test_basic(self): self.run_stage(self.args) # Load the output file vfr = tk_io.VariantFileReader(os.path.join(job_dir,"default.vcf")) for r in vfr.record_getter(): pos = tk_io.get_record_pos(r) barcodes = tk_io.get_record_barcodes(r) if pos == 26357747 or pos == 26357748: print barcodes assert(barcodes[1][0] =='1-ATAGGAGTTCAGGG_63') print tk_io.get_record_alt_allele_counts(r) assert(int(tk_io.get_record_alt_allele_counts(r)[0]) in [31,32]) assert(int(tk_io.get_record_ref_allele_count(r)) == 0) if pos == 26501280: print barcodes assert(barcodes[0][0] == '1-TGAAGACATAACCC_61_61') assert(int(r.INFO['POSTHPC'][0]) == 10)
def main(args, outs): def real_file(f): return f is not None and os.path.isfile(f) and os.path.getsize(f) > 0 precalled = False called = False if real_file(args.precalled_chunk): precalled = True if args.precalled_no_chunk: return elif real_file(args.chunk_input): called = True if called: input_vcf = "canonicalized.vcf" canonicalize(args.chunk_input, input_vcf) elif precalled: canonicalize(args.precalled_chunk, "canonicalized.vcf") input_vcf = outs.default+"tmp.vcf" vcf_reader = tk_io.VariantFileReader("canonicalized.vcf") with open(input_vcf, "w") as file_write: output = tk_io.VariantFileWriter(file_write, template_file=open("canonicalized.vcf")) for record in vcf_reader.record_getter(): (bad, message) = tk_io.is_record_bad_variant(record) if bad: try: record_strified = str(record) except: record_strified = "<error displaying record>" martian.exit("error on vcf record: "+record_strified+", "+message) if record.QUAL == None: record.QUAL = 0 info_fields = {key: record.INFO.get(key) for key in VCF_WHITE_LIST_INFO_FIELDS if key in record.INFO} record.INFO = info_fields sample_call = tk_io.get_record_sample_call(record) data = sample_call.data data_dict = data._asdict() if "GT" in data_dict: new_sample_vals = [data_dict["GT"]] new_format = "GT" new_fields = ["GT"] else: new_sample_vals = ["./."] new_format = "GT" new_fields = ["GT"] data_instantiator = vcf.model.make_calldata_tuple(new_fields) data = data_instantiator(*new_sample_vals) sample_call.data = data record.samples[0] = sample_call record.FORMAT = new_format output.write_record(record) else: outs.default = None return reference_pyfasta = tenkit.reference.open_reference(args.reference_path) chunk_record_size = 10000 vcf_reader = tk_io.VariantFileReader(input_vcf) # Check whether to write GL field v = vcf.Reader(open(input_vcf)) write_gl = v.formats.has_key('GL') bam = tk_bam.create_bam_infile(args.bam) outfile = outs.default if not precalled: outfile = outs.default+"tmp2.vcf" with open(outfile, "w") as file_write: new_source = "10X/pipelines/stages/snpindels/attach_bcs_snpindels %s" % martian.get_pipelines_version() new_formats = [("BX", ".", "String", "Barcodes and Associated Qual-Scores Supporting Alleles")] AO = ('AO','.','Integer','Alternate allele observed count', None, None) RO = ('RO','1','Integer','Reference allele observed count', None, None) post_homopolymer_counts_field = ('POSTHPC','.', 'Integer', 'Postvariant homopolymer count', None, None) post_homopolymer_base_field = ('POSTHPB','.', 'Character','Postvariant homopolymer base', None, None) post_dinucleotide_base_field = ('POSTDNB','.', 'String', 'Post variant dinucleotide repeat sequence', None, None) post_dinucleotide_counts_field = ('POSTDNC','.','Integer','Post variant dinucleotide repeat count', None, None) post_trinucleotide_base_field = ('POSTTNB','.','String','Post variant trinucleotide repeat sequence', None, None) post_trinucleotide_counts_field = ('POSTTNC','.','Integer','Post variant trinucleotide repeat count', None, None) mean_map_alt = ('MUMAP_ALT', '.', 'Float', 'Mean mapping scores of alt alleles', None, None) mean_map_ref = ('MUMAP_REF', '1', 'Float', 'Mean mapping score of ref allele', None, None) rescued = ('RESCUED','.', 'Integer','How many reads were rescued via cross barcode mapq correction', None, None) not_rescued = ('NOT_RESCUED','.','Integer', 'How many reads were not rescued via cross barcode mapq correction', None, None) mean_molecule_difference = ("MMD", '.', 'Float', 'Mean molecule divergence from reference per read', None, None) haplocalled = ("HAPLOCALLED", "1", "Integer", "1 for variants that were called after phasing via splitting the bam into its component haplotypes and calling variants in haploid mode", None, None) extra_fields = [post_homopolymer_counts_field, post_homopolymer_base_field, mean_map_ref, mean_map_alt, AO, RO, mean_molecule_difference, rescued, not_rescued, post_dinucleotide_base_field, post_dinucleotide_counts_field, post_trinucleotide_base_field, post_trinucleotide_counts_field, haplocalled] output = tk_io.VariantFileWriter(file_write, template_file=open(input_vcf), new_source=new_source, new_info_fields=extra_fields, new_format_fields=new_formats) record_list = [] for record in vcf_reader.record_getter(): if len(record_list) == chunk_record_size: for record_out in record_list: if tk_io.get_record_passes_filters(record_out): populate_fields(record_out, bam, reference_pyfasta, args) for record_out in record_list: output.write_record(record_out) record_list = [] record_list.append(record) for record_out in record_list: if tk_io.get_record_passes_filters(record_out): populate_fields(record_out, bam, reference_pyfasta, args) for record_out in record_list: output.write_record(record_out) if not precalled and args.haploid_merge is not None: output_filename = outs.default+"tmp3.vcf" merge_haploid(outfile, args.haploid_merge, args.locus, output_filename, bam, reference_pyfasta, args, write_gl) outfile = output_filename # Filter variants if not precalled: last_file = outfile for filter in VARIANT_CALL_FILTER: with open(outs.default+"tmp.vcf",'w') as output_file: filt = VARIANT_CALL_FILTER[filter] tenkit.log_subprocess.check_call(['bcftools','filter','-O', 'v','--soft-filter',filter,'-e',filt,'-m','\'+\'', last_file],stdout=output_file) tenkit.log_subprocess.check_call(['mv', outs.default+"tmp.vcf", outs.default+"tmp2.vcf"]) last_file = outs.default+"tmp2.vcf" tenkit.log_subprocess.check_call(['mv', last_file, outs.default])
def check_concordance(pb_left, pb_right, vcf, max_allowable_overhang_length=1000, max_allowable_overhang_variants=2, filter_trivial=True): ''' Compare two sets of phase blocks - what's shared and what's different. The max_allowable_overhang parameters allow one to ignore minor differences. ''' # hack: change chrom names to numbers to get around pandas 0.15.2 bug when merging categoricals # (see https://github.com/pydata/pandas/issues/9426) all_chroms = pandas.Categorical( pb_left.chrom.append(pb_right.chrom).drop_duplicates()) chrom_left_codes = pandas.Categorical( pb_left.chrom).set_categories(all_chroms).codes chrom_right_codes = pandas.Categorical( pb_right.chrom).set_categories(all_chroms).codes pb_left_tmp = pandas.DataFrame({ 'chrom': chrom_left_codes, 'phase_set': pb_left.phase_set, 'start_left': pb_left.start, 'end_left': pb_left.end }) pb_right_tmp = pandas.DataFrame({ 'chrom': chrom_right_codes, 'phase_set': pb_right.phase_set, 'start_right': pb_right.start, 'end_right': pb_right.end }) pb_merged = pandas.merge(pb_left_tmp, pb_right_tmp, on=['chrom', 'phase_set'], how='outer') if filter_trivial: pb_merged = pb_merged[(pb_merged.end_right - pb_merged.start_right > 1) | (pb_merged.end_left - pb_merged.start_left > 1)].reset_index() # map codes back to chrom names pb_chroms = pandas.Categorical.from_codes(pb_merged.chrom, all_chroms.categories) pb_merged.chrom = pb_chroms # split into shared/non-shared phase sets pb_merged_shared = pb_merged[pb_merged.start_left.notnull() & pb_merged.start_right.notnull()] pb_merged_left_only = pb_merged[pb_merged.start_right.isnull()] pb_merged_right_only = pb_merged[pb_merged.start_left.isnull()] # look at overlaps in the shared sets start_max = pb_merged_shared.start_left.combine( pb_merged_shared.start_right, max) end_min = pb_merged_shared.end_left.combine(pb_merged_shared.end_right, min) overlap = map(lambda x: max(x, 0), end_min - start_max) non_overlap = (pb_merged_shared.end_left - pb_merged_shared.start_left) + ( pb_merged_shared.end_right - pb_merged_shared.start_right) - map( lambda x: 2 * x, overlap) discord = (non_overlap > max_allowable_overhang_length) pb_merged_shared_concordant = pb_merged_shared[~discord] pb_merged_shared_discordant = pb_merged_shared[discord] # ignore cases with only a few overhang variants (probably due to off-by-one errors) if len(pb_merged_shared_discordant) > 0: vfr = tk_io.VariantFileReader(vcf) pb_merged_shared_discordant = pb_merged_shared_discordant[ pb_merged_shared_discordant.apply( lambda x: too_many_overhang_variants( x, vfr, max_allowable_overhang_variants), axis=1)] return (pb_merged_shared_concordant, pb_merged_shared_discordant, pb_merged_left_only, pb_merged_right_only)
def mk_var_tuples(fn, targets, locus): reader = tk_io.VariantFileReader(fn) var_iter = tk_io.get_variant_iterator_pos(reader, targets, locus) var_tuples = variant_tuples(var_iter) return var_tuples
def main(args, outs): vc_mode, _, _, _ = tk_io.get_vc_mode(args.vc_precalled, args.vc_mode) (chrom, start, stop) = tk_io.get_locus_info(args.locus) chrom = str(chrom) if chrom in ['chrM', 'MT', 'M'] or (args.sex.lower() in ["f", "female"] and chrom in ["chrY", "Y"]): return fragment_barcode_info = pysam.Tabixfile(args.fragment_phasing) AH_0_BH_0 = ( 'AH_0_BH_0', '1', 'Integer', 'Number of barcodes that have been called as supporting haplotype 0 which are on reads that have support for the allele which has been phased as haplotype 0' ) AH_1_BH_1 = ( 'AH_1_BH_1', '1', 'Integer', 'Number of barcodes that have been called as supporting haplotype 1 which are on reads that have support for the allele which has been phased as haplotype 1' ) AH_0_BH_1 = ( 'AH_0_BH_1', '1', 'Integer', 'Number of barcodes that have been called as supporting haplotype 0 which are on reads that have support for the allele which has been phased as haplotype 1' ) AH_1_BH_0 = ( 'AH_1_BH_0', '1', 'Integer', 'Number of barcodes that have been called as supporting haplotype 1 which are on reads that have support for the allele which has been phased as haplotype 0' ) BX_HAP_OR = ( 'BX_HAP_OR', '1', 'Float', "Barcode aware haplotype filtering score (log odds ratio currently)") BARCODE_AWARE_FILTER = [( "BARCODE_AWARE_FILTER", "Uses haplotype information from the fragments and the alleles to filter some variants that are not consistent with haplotype (ie variants should have most of their allele haplotype 0 alleles coming from barcodes whose fragments are haplotype 0 etc)" )] extra_fields = [AH_0_BH_0, AH_1_BH_1, AH_0_BH_1, AH_1_BH_0, BX_HAP_OR] input_variants = tk_io.VariantFileReader(args.variants) with open(outs.default.strip(".gz"), 'w') as output_file: output_variants = tk_io.VariantFileWriter( output_file, template_file=open(args.variants, 'r'), new_info_fields=extra_fields, new_filters=BARCODE_AWARE_FILTER) variant_iterator = tk_io.get_variant_iterator_pos( input_variants, None, args.locus) for record in variant_iterator: sample = record.samples[0] ref = tk_io.get_record_ref(record) alt_alleles = tk_io.get_record_alt_alleles(record) if not tk_io.get_record_passes_filters(record): output_variants.write_record(record) continue if len(sample.gt_alleles) > 1: genotype_1 = int(sample.gt_alleles[0]) genotype_2 = int(sample.gt_alleles[1]) if genotype_1 == genotype_2: output_variants.write_record(record) continue #homozygous, can't filter this way else: output_variants.write_record(record) continue #homozygous, can't filter this way chrom = tk_io.get_record_chrom(record) if not chrom == "chrM": variant_barcode_info = load_variant_barcode_phasing_info( record, fragment_barcode_info) if not barcode_aware_filter(record, variant_barcode_info): if record.FILTER is None: record.FILTER = [] if tk_io.get_var_type(ref, alt_alleles[0]) == "S" and ( (vc_mode == 'call') or (vc_mode == "precalled_plus" and "TENX" in record.INFO)): record.FILTER.append("BARCODE_AWARE_FILTER") output_variants.write_record(record)