def test_string_or_vcftype(vcfdir): assert (trh.HasLengthAltGenotypes("gangstr") == trh.HasLengthAltGenotypes( trh.VcfTypes.gangstr)) assert (trh.HasLengthRefGenotype("gangstr") == trh.HasLengthRefGenotype( trh.VcfTypes.gangstr)) assert (trh.MayHaveImpureRepeats("gangstr") == trh.MayHaveImpureRepeats( trh.VcfTypes.gangstr)) reset_vcfs(vcfdir) assert (trh.HarmonizeRecord( "gangstr", next(gangstr_vcf)).GetMaxAllele() == len("tctgtctgtctg") / len("tctg")) assert (trh.HarmonizeRecord( trh.VcfTypes.gangstr, next(gangstr_vcf)).GetMaxAllele() == len("aaaacaaaacaaaacaaaac") / len("aaaac"))
def test_wrong_vcftype(vcfdir): # an iterator that includes both tr caller types # and error file types for correct_type in trh.VcfTypes: reset_vcfs(vcfdir) for incorrect_type in all_types(): if incorrect_type == correct_type: # make sure the incorrect_type is actually incorrect continue invcf = get_vcf(incorrect_type) with pytest.raises(TypeError): print(correct_type, incorrect_type) trh.TRRecordHarmonizer(invcf, vcftype=correct_type) reset_vcfs(vcfdir) for incorrect_type in all_types(): if incorrect_type == correct_type: # make sure the incorrect_type is actually incorrect continue invcf = get_vcf(incorrect_type) record = next(invcf) with pytest.raises(TypeError): print(correct_type, incorrect_type) trh.HarmonizeRecord(correct_type, record)
def __call__(self, record): trrecord = trh.HarmonizeRecord(self.vcftype, record) het = utils.GetHeterozygosity( trrecord.GetAlleleFreqs(uselength=self.uselength)) if het > self.threshold: return het return None
def __call__(self, record): trrecord = trh.HarmonizeRecord(self.vcftype, record) allele_freqs = trrecord.GetAlleleFreqs(uselength=self.uselength) genotype_counts = trrecord.GetGenotypeCounts(uselength=self.uselength) hwep = utils.GetHardyWeinbergBinomialTest(allele_freqs, genotype_counts) if hwep < self.threshold: return hwep else: return None
def main_helper(output, str_imputation_run_name, chrom, all_white_brits_fname): vcf_fname = (f'{ukb}/str_imputed/runs/{str_imputation_run_name}/' f'vcfs/annotated_strs/chr{chrom}.vcf.gz') vcf = cyvcf2.VCF(vcf_fname) subset_samples = [] with open(all_white_brits_fname) as samp_file: next(samp_file) for line in samp_file: subset_samples.append(line.strip()) all_samples = [l[0] for l in np.char.split(vcf.samples, '_')] samples = np.isin(all_samples, subset_samples) output.write( 'chr\tpos\tallele_dist\tentropy\theterozygosity\tmultiallelicness\n') for record in vcf: if record.INFO.get('PERIOD') is None: continue trrecord = trh.HarmonizeRecord(vcfrecord=record, vcftype='beagle-hipstr') len_alleles = [trrecord.ref_allele_length ] + trrecord.alt_allele_lengths total_subset_dosages = {len_: 0 for len_ in np.unique(len_alleles)} for p in (1, 2): ap = trrecord.format[f'AP{p}'] total_subset_dosages[len_alleles[0]] += \ np.sum(np.maximum(0, 1 - np.sum(ap[samples, :], axis=1))) for i in range(ap.shape[1]): total_subset_dosages[len_alleles[i + 1]] += np.sum(ap[samples, i]) for len_ in total_subset_dosages: total_subset_dosages[len_] /= np.sum(samples) * 2 entropy = scipy.stats.entropy(list(total_subset_dosages.values()), base=2) heterozygosity = 1 - np.sum(val**2 for val in total_subset_dosages.values()) multiallelicness = sum(sorted(total_subset_dosages.values())[:-2]) output.write(trrecord.chrom + '\t' + str(trrecord.pos) + '\t' + str(total_subset_dosages) + '\t' + str(entropy) + '\t' + str(heterozygosity) + '\t' + str(multiallelicness) + '\n')
def test_HarmonizeRecord(vcfdir): reset_vcfs(vcfdir) # Unknown type with pytest.raises(ValueError): trh.HarmonizeRecord("foo", next(snps_vcf)) # Gangstr gangstr_trh = trh.TRRecordHarmonizer(gangstr_vcf) tr_rec1 = next(iter(gangstr_trh)) assert tr_rec1.ref_allele == 'tctgtctgtctg'.upper() assert tr_rec1.alt_alleles == [] assert tr_rec1.motif == 'tctg'.upper() assert not tr_rec1.HasFullStringGenotypes() assert not tr_rec1.HasFabricatedRefAllele() assert not tr_rec1.HasFabricatedAltAlleles() tr_rec2 = next(iter(gangstr_trh)) tr_rec3 = next(iter(gangstr_trh)) assert (tr_rec3.ref_allele == 'tgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtg'.upper()) assert (tr_rec3.alt_alleles == [ 'tgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtgtg'.upper() ]) assert tr_rec3.motif == 'tg'.upper() # hipstr hipstr_trh = trh.TRRecordHarmonizer(hipstr_vcf) str_iter = iter(hipstr_trh) tr_rec1 = next(str_iter) assert tr_rec1.ref_allele == 'GGTGGTGGTGGGGGCGGTGGGGGTGGTG' assert tr_rec1.alt_alleles == ['GGTGGTGGTGGGGGCGGTGGTGGTGCTG'] assert tr_rec1.motif == 'GGT' assert tr_rec1.record_id == 'STR_2' assert not tr_rec1.HasFullStringGenotypes() assert not tr_rec1.HasFabricatedRefAllele() assert not tr_rec1.HasFabricatedAltAlleles() tr_rec2 = next(str_iter) tr_rec3 = next(str_iter) assert tr_rec3.ref_allele == 'TTTTTTTTTTTTTTT' assert tr_rec3.alt_alleles == [] assert tr_rec3.motif == 'T'.upper() assert tr_rec3.record_id == 'STR_4' record = next(str_iter) while record.record_id != "STR_125": record = next(str_iter) assert record.HasFullStringGenotypes() # TODO this isn't really the correct behavior - # we're trimming off an extra repeat from the alt allele assert record.full_alleles == ( "TGCATATATGTATAATATATATTATATATGGA", ["TCCATATATGCATAATATATATTATATATATG"], ) assert (record.ref_allele == "ATATATGTATAATATATATTATATAT") assert (record.alt_alleles == ["ATATATGCATAATATATATTATATAT"]) # popstr popstr_trh = trh.TRRecordHarmonizer(popstr_vcf) tr_rec1 = next(iter(popstr_trh)) assert tr_rec1.ref_allele == 'GGGGGGGCGGGGGGGGGG' assert tr_rec1.alt_alleles == ['G' * 14, 'G' * 17] assert tr_rec1.motif == 'G' assert tr_rec1.record_id == 'chr21:5020351:M' assert not tr_rec1.HasFullStringGenotypes() assert not tr_rec1.HasFabricatedRefAllele() assert tr_rec1.HasFabricatedAltAlleles() tr_rec2 = next(iter(popstr_trh)) tr_rec3 = next(iter(popstr_trh)) assert tr_rec3.ref_allele == 'TTTTTTTTTTTTTTTTTTTTTT' assert tr_rec3.alt_alleles == ['T' * 21] assert tr_rec3.motif == 'T' assert tr_rec3.record_id == 'chr21:5031126:M' # advntr advntr_trh = trh.TRRecordHarmonizer(advntr_vcf) tr_rec1 = next(iter(advntr_trh)) assert tr_rec1.ref_allele == 'GCGCGGGGCGGGGCGCGGGGCGGGGCGCGGGGCGGG' assert tr_rec1.alt_alleles == [ 'GCGCGGGGCGGGGCGCGGGGCGGG', 'GCGCGGGGCGGGGCGCGGGGCGGGGCGCGGGGCGGGGCGCGGGGCGGGGCGCGGGGCGGG' ] assert tr_rec1.motif == 'GCGCGGGGCGGG' assert not tr_rec1.HasFullStringGenotypes() assert not tr_rec1.HasFabricatedRefAllele() assert not tr_rec1.HasFabricatedAltAlleles() # advntr eh_trh = trh.TRRecordHarmonizer(eh_vcf) tr_rec1 = next(iter(eh_trh)) motif = 'CAG' assert tr_rec1.ref_allele == motif * 19 assert tr_rec1.alt_alleles == [motif * 16, motif * 18] assert tr_rec1.motif == motif assert tr_rec1.record_id == 'HTT' assert not tr_rec1.HasFullStringGenotypes() assert tr_rec1.HasFabricatedRefAllele() assert tr_rec1.HasFabricatedAltAlleles()
def main(args): if not os.path.exists(os.path.dirname(os.path.abspath(args.out))): common.WARNING( "Error: The directory which contains the output location {} does" " not exist".format(args.out)) return 1 if os.path.isdir(args.out) and args.out.endswith(os.sep): common.WARNING("Error: The output location {} is a " "directory".format(args.out)) return 1 ### Check and load VCF files ### vcfreaders = utils.LoadReaders([args.vcf1, args.vcf2], checkgz=True, region=args.region) if vcfreaders is None or len(vcfreaders) != 2: return 1 contigs = vcfreaders[0].contigs chroms = list(contigs) ### Load shared samples ### samples = mergeutils.GetSharedSamples(vcfreaders) if len(samples) == 0: common.WARNING("No shared smaples found between vcf readers") return 1 if args.samples: usesamples = set( [item.strip() for item in open(args.samples, "r").readlines()]) samples = list(set(samples).intersection(usesamples)) if len(samples) == 0: common.WARNING("No shared samples found between files") return 1 ### Determine FORMAT fields we should look for ### if args.stratify_file is not None and args.stratify_file not in [0, 1, 2]: common.MSG("--stratify-file must be 0,1, or 2") return 1 format_fields, format_binsizes = GetFormatFields(args.stratify_fields, args.stratify_binsizes, args.stratify_file, vcfreaders) ### Keep track of data to summarize at the end ### results_dir = { "chrom": [], "start": [], "period": [], "sample": [], "gtstring1": [], "gtstring2": [], "gtsum1": [], "gtsum2": [], "metric-conc-seq": [], "metric-conc-len": [], } for ff in format_fields: results_dir[ff + "1"] = [] results_dir[ff + "2"] = [] vcftype1 = trh.GetVCFType(vcfreaders[0], args.vcftype1) vcftype2 = trh.GetVCFType(vcfreaders[1], args.vcftype2) ### Walk through sorted readers, merging records as we go ### current_records = [next(reader) for reader in vcfreaders] is_min = mergeutils.GetMinRecords(current_records, chroms) done = mergeutils.DoneReading(current_records) num_records = 0 while not done: if any([item is None for item in current_records]): break if args.numrecords is not None and num_records >= args.numrecords: break if args.verbose: mergeutils.DebugPrintRecordLocations(current_records, is_min) if mergeutils.CheckMin(is_min): return 1 if all([is_min]): if (current_records[0].CHROM == current_records[1].CHROM and \ current_records[0].POS == current_records[1].POS): UpdateComparisonResults(trh.HarmonizeRecord(vcftype1, current_records[0]), \ trh.HarmonizeRecord(vcftype2, current_records[1]), \ format_fields, samples, results_dir) current_records = mergeutils.GetNextRecords(vcfreaders, current_records, is_min) is_min = mergeutils.GetMinRecords(current_records, chroms) done = mergeutils.DoneReading(current_records) num_records += 1 ### Load all results to a dataframe and output full results ### data = pd.DataFrame(results_dir) data.to_csv(args.out + "-callcompare.tab", sep="\t", index=False) ### Overall metrics ### OutputOverallMetrics(data, format_fields, format_binsizes, args.stratify_file, args.period, args.out) if not args.noplot: OutputBubblePlot(data, args.period, args.out, minval=args.bubble_min, maxval=args.bubble_max) ### Per-locus metrics ### OutputLocusMetrics(data, args.out, args.noplot) ### Per-sample metrics ### OutputSampleMetrics(data, args.out, args.noplot) return 0
def main(args): # Load VCF file invcf = utils.LoadSingleReader(args.vcf, checkgz=False) if invcf is None: return 1 if not os.path.exists(os.path.dirname(os.path.abspath(args.out))): common.WARNING( "Error: The directory which contains the output location {} does" " not exist".format(args.out)) return 1 if os.path.isdir(args.out) and args.out.endswith(os.sep): common.WARNING("Error: The output location {} is a " "directory".format(args.out)) return 1 # Set up record harmonizer and infer VCF type vcftype = trh.InferVCFType(invcf, args.vcftype) # Check filters all make sense if not CheckFilters(invcf, args, vcftype): return 1 # Set up locus-level filter list try: filter_list = BuildLocusFilters(args, vcftype) except ValueError: return 1 filter_list = BuildLocusFilters(args, vcftype) invcf.filters = {} for f in filter_list: short_doc = f.__doc__ or '' short_doc = short_doc.split('\n')[0].lstrip() invcf.filters[f.filter_name()] = _Filter(f.filter_name(), short_doc) # Set up call-level filters call_filters = BuildCallFilters(args) # Add new FORMAT fields if "FILTER" not in invcf.formats: invcf.formats["FILTER"] = _Format("FILTER", 1, "String", "Call-level filter") # Add new INFO fields invcf.infos["AC"] = _Info("AC", -1, "Integer", "Alternate allele counts", source=None, version=None) invcf.infos["REFAC"] = _Info("REFAC", 1, "Integer", "Reference allele count", source=None, version=None) invcf.infos["HET"] = _Info("HET", 1, "Float", "Heterozygosity", source=None, version=None) invcf.infos["HWEP"] = _Info("HWEP", 1, "Float", "HWE p-value for obs. vs. exp het rate", source=None, version=None) invcf.infos["HRUN"] = _Info("HRUN", 1, "Integer", "Length of longest homopolymer run", source=None, version=None) # Set up output files if not os.path.exists(os.path.dirname(os.path.abspath(args.out))): common.WARNING("Output directory does not exist") return 1 outvcf = MakeWriter(args.out + ".vcf", invcf, " ".join(sys.argv)) if outvcf is None: return 1 # Set up sample info all_reasons = GetAllCallFilters(call_filters) sample_info = {} for s in invcf.samples: sample_info[s] = {"numcalls": 0, "totaldp": 0} for r in all_reasons: sample_info[s][r] = 0 # Set up locus info loc_info = {"totalcalls": 0, "PASS": 0} for filt in filter_list: loc_info[filt.filter_name()] = 0 # Go through each record record_counter = 0 while True: try: record = next(invcf) except IndexError: common.WARNING( "Skipping TR that couldn't be parsed by PyVCF. Check VCF format" ) if args.die_on_warning: return 1 except StopIteration: break if args.verbose: common.MSG("Processing %s:%s" % (record.CHROM, record.POS)) record_counter += 1 if args.num_records is not None and record_counter > args.num_records: break # Call-level filters record = ApplyCallFilters(record, invcf, call_filters, sample_info) # Locus-level filters record.FILTER = None output_record = True for filt in filter_list: if filt(record) == None: continue if args.drop_filtered: output_record = False break record.add_filter(filt.filter_name()) loc_info[filt.filter_name()] += 1 if args.drop_filtered: if record.call_rate == 0: output_record = False if output_record: trrecord = trh.HarmonizeRecord(vcftype, record) # Recalculate locus-level INFO fields record.INFO["HRUN"] = utils.GetHomopolymerRun(record.REF) if record.num_called > 0: allele_freqs = trrecord.GetAlleleFreqs( uselength=args.use_length) genotype_counts = trrecord.GetGenotypeCounts( uselength=args.use_length) record.INFO["HET"] = utils.GetHeterozygosity(allele_freqs) record.INFO["HWEP"] = utils.GetHardyWeinbergBinomialTest( allele_freqs, genotype_counts) record.INFO["AC"] = [ int(item * (3 * record.num_called)) for item in record.aaf ] record.INFO["REFAC"] = int( (1 - sum(record.aaf)) * (2 * record.num_called)) else: record.INFO["HET"] = -1 record.INFO["HWEP"] = -1 record.INFO["AC"] = [0] * len(record.ALT) record.INFO["REFAC"] = 0 # Recalc filter if record.FILTER is None and not args.drop_filtered: record.FILTER = "PASS" loc_info["PASS"] += 1 loc_info["totalcalls"] += record.num_called # Output the record outvcf.write_record(record) # Output log info WriteSampLog(sample_info, all_reasons, args.out + ".samplog.tab") WriteLocLog(loc_info, args.out + ".loclog.tab") return 0
def load_strs(imputation_run_name: str, region: str, samples: np.ndarray, details: bool = True, var_subset: Optional[Set[int]] = None, hardcalls=False): """ Iterate over a region returning genotypes at STR loci. First yield is a tuple of names of the fields in details. Every subsequent yield is described in the yields section below. Parameters ---------- imputation_run_name: which imputation run to load genotypes from? region: chr:start-end samples: A boolean array of length nsamples determining which samples are included (True) and which are not Yields ------ dosages: Dict[float, np.ndarray] A dictionary from unique length alleles to 2D arrays of size (n_samples, 2) which contain the dosages of those alleles for each haplotype Length dosage are measured in number of repeats. None if locus_filtered is not None If hardcalls, then instead of that just an array nx2 of length alleles unique_alleles: np.ndarray Array of unique length alleles (measured in number of repeats), same length as the dosages dict chrom: str e.g. '13' pos: int locus_filtered: None if the locus is not filtered, otherwise a string explaining why. 'MAC<20' if the minor allele dosage is less than 20 after sample subsetting, per plink's standard. None if hardcalls. locus_details: tuple of strings with the same length as the first yield with the corresponding order. None if hardcalls. Notes ----- Hardcalls mentioned in the locus details are phased hardcalls, and in some corner cases will not correspond to the maximum likelihood unphased allele. """ if hardcalls: assert var_subset is not None and not details chrom, region_poses = region.split(':') region_start, _ = [int(pos) for pos in region_poses.split('-')] vcf_fname = (f'{ukb}/str_imputed/runs/{imputation_run_name}/' f'vcfs/annotated_strs/chr{chrom}.vcf.gz') vcf = cyvcf2.VCF(vcf_fname) if details: yield ('motif', 'period', 'ref_len', 'total_per_allele_dosages', 'total_hardcall_alleles', 'total_hardcall_genotypes', 'subset_total_per_allele_dosages', 'subset_total_hardcall_alleles', 'subset_total_hardcall_genotypes', 'subset_het', 'subset_entropy', 'subset_HWEP', 'subset_allele_dosage_r2') for record in vcf(region): if record.POS < region_start: # records that overlap this region but started before this region # should be considered part of the pervious region and not returned here continue if record.INFO.get('PERIOD') is None: # there are a few duplicate loci which I didn't handle # properly, this identifies and removes them continue if var_subset is not None and record.POS not in var_subset: continue trrecord = trh.HarmonizeRecord(vcfrecord=record, vcftype='beagle-hipstr') len_alleles = [trrecord.ref_allele_length ] + trrecord.alt_allele_lengths len_alleles = [ round(allele_len, allele_len_precision) for allele_len in len_alleles ] if hardcalls: yield (trrecord.GetLengthGenotypes()[samples, :-1], np.unique(len_alleles), trrecord.chrom, trrecord.pos, None, None) continue if details: total_dosages = {_len: 0 for _len in np.unique(len_alleles)} for p in (1, 2): ap = trrecord.format[f'AP{p}'] total_dosages[len_alleles[0]] += np.sum( np.maximum(0, 1 - np.sum(ap, axis=1))) for i in range(ap.shape[1]): total_dosages[len_alleles[i + 1]] += np.sum(ap[:, i]) total_hardcall_alleles = clean_len_alleles( trrecord.GetAlleleCounts()) total_hardcall_genotypes = clean_len_allele_pairs( trrecord.GetGenotypeCounts()) if isinstance(samples, slice): assert samples == slice(None) n_subset_samples = trrecord.GetNumSamples() else: n_subset_samples = int(np.sum(samples)) subset_dosage_gts = { _len: np.zeros((n_subset_samples, 2)) for _len in np.unique(len_alleles) } for p in (1, 2): # todo genotype dosages ap = trrecord.format[f'AP{p}'] subset_dosage_gts[len_alleles[0]][:, (p-1)] += \ np.maximum(0, 1 - np.sum(ap[samples, :], axis=1)) for i in range(ap.shape[1]): subset_dosage_gts[len_alleles[i + 1]][:, (p - 1)] += ap[samples, i] subset_total_dosages = { _len: np.sum(subset_dosage_gts[_len]) for _len in subset_dosage_gts } if details: subset_total_hardcall_alleles = clean_len_alleles( trrecord.GetAlleleCounts(samples)) subset_total_hardcall_genotypes = clean_len_allele_pairs( trrecord.GetGenotypeCounts(samples)) subset_hardcall_allele_freqs = clean_len_alleles( trrecord.GetAlleleFreqs(samples)) subset_het = utils.GetHeterozygosity(subset_hardcall_allele_freqs) subset_entropy = utils.GetEntropy(subset_hardcall_allele_freqs) subset_hwep = utils.GetHardyWeinbergBinomialTest( subset_hardcall_allele_freqs, subset_total_hardcall_genotypes) # https://www.cell.com/ajhg/fulltext/S0002-9297(09)00012-3#app1 # Browning, Brian L., and Sharon R. Browning. "A unified approach to genotype imputation and haplotype-phase inference for large data sets of trios and unrelated individuals." The American Journal of Human Genetics 84.2 (2009): 210-223. # appendix 1 subset_allele_dosage_r2 = {} subset_hardcalls = np.around( trrecord.GetLengthGenotypes()[samples, :-1], allele_len_precision) for length in len_alleles: # calculate allele dosage r**2 for this length if length in subset_allele_dosage_r2: continue calls = subset_hardcalls == length subset_allele_dosage_r2[length] = np.corrcoef( calls.reshape(-1), subset_dosage_gts[length].reshape(-1))[0, 1]**2 locus_details = (trrecord.motif, str(len(trrecord.motif)), str( round(trrecord.ref_allele_length, allele_len_precision)), dict_str( round_vals(total_dosages, dosage_precision)), dict_str(total_hardcall_alleles), dict_str(total_hardcall_genotypes), dict_str( round_vals(subset_total_dosages, dosage_precision)), dict_str(subset_total_hardcall_alleles), dict_str(subset_total_hardcall_genotypes), str(subset_het), str(subset_entropy), str(subset_hwep), dict_str( round_vals(subset_allele_dosage_r2, r2_precision))) else: locus_details = None mac = list(subset_total_dosages.values()) mac.pop(np.argmax(mac)) if np.sum(mac) < 20: yield (None, np.unique(len_alleles), trrecord.chrom, trrecord.pos, 'MAC<20', locus_details) continue yield (subset_dosage_gts, np.unique(len_alleles), trrecord.chrom, trrecord.pos, None, locus_details)
merged_arr = utils.merge_arrays(samples_array, pheno_data) unfiltered_subset = ~np.isnan(merged_arr[:, 1]) n_samples = np.sum(unfiltered_subset) vcf = cyvcf2.VCF(args.imputed_vcf) found_rec = False for record in vcf(f'{args.chrom}:{args.pos}-{args.pos}'): if record.POS < args.pos: continue if record.INFO.get('PERIOD') is None: continue assert not found_rec found_rec = True trrecord = trh.HarmonizeRecord(vcfrecord=record, vcftype='beagle-hipstr') len_alleles = [trrecord.ref_allele_length] + trrecord.alt_allele_lengths len_alleles = [round(allele_len, 2) for allele_len in len_alleles] ap1 = trrecord.format['AP1'] ap1 = np.concatenate((1 - np.sum(ap1, axis=1).reshape(-1, 1), ap1), axis=1) ap2 = trrecord.format['AP2'] ap2 = np.concatenate((1 - np.sum(ap2, axis=1).reshape(-1, 1), ap2), axis=1) # TODO this needs better testing subset_summed_dosages = {} for aidx1, len_allele1 in enumerate(len_alleles): for aidx2, len_allele2 in enumerate(len_alleles): summed_len = len_allele1 + len_allele2 if summed_len not in subset_summed_dosages:
def main(args): if not os.path.exists(args.vcf): common.WARNING("Error: %s does not exist" % args.vcf) return 1 if not os.path.exists(os.path.dirname(os.path.abspath(args.out))): common.WARNING( "Error: The directory which contains the output location {} does" " not exist".format(args.out)) return 1 if os.path.isdir(args.out) and args.out.endswith(os.sep): common.WARNING("Error: The output location {} is a " "directory".format(args.out)) return 1 # Load samples sample_lists = [] sample_prefixes = [] if args.samples: sfiles = args.samples.split(",") if args.sample_prefixes: sample_prefixes = args.sample_prefixes.split(",") else: sample_prefixes = [str(item) for item in range(1, len(sfiles) + 1)] if len(sfiles) != len(sample_prefixes): common.MSG("--sample-prefixes must be same length as --samples") return 1 for sf in sfiles: sample_lists.append( [item.strip() for item in open(sf, "r").readlines()]) invcf = utils.LoadSingleReader(args.vcf, checkgz=False) if invcf is None: return 1 if args.vcftype != 'auto': vcftype = trh.VcfTypes[args.vcftype] else: vcftype = trh.InferVCFType(invcf) header = ["chrom", "start", "end"] if args.thresh: header.extend(GetHeader("thresh", sample_prefixes)) if args.afreq: header.extend(GetHeader("afreq", sample_prefixes)) if args.acount: header.extend(GetHeader("acount", sample_prefixes)) if args.hwep: header.extend(GetHeader("hwep", sample_prefixes)) if args.het: header.extend(GetHeader("het", sample_prefixes)) if args.mean: header.extend(GetHeader("mean", sample_prefixes)) if args.mode: header.extend(GetHeader("mode", sample_prefixes)) if args.var: header.extend(GetHeader("var", sample_prefixes)) if args.numcalled: header.extend(GetHeader("numcalled", sample_prefixes)) if args.out == "stdout": if args.plot_afreq: common.MSG("Cannot use --out stdout when generating plots") return 1 outf = sys.stdout else: outf = open(args.out + ".tab", "w") outf.write("\t".join(header) + "\n") if args.region: if not os.path.isfile(args.vcf + ".tbi"): common.MSG("Make sure %s is bgzipped and indexed" % args.vcf) return 1 regions = invcf.fetch(args.region) else: regions = invcf num_plotted = 0 for record in regions: trrecord = trh.HarmonizeRecord(vcftype, record) if args.plot_afreq and num_plotted <= MAXPLOTS: PlotAlleleFreqs(trrecord, args.out, samplelists=sample_lists, sampleprefixes=sample_prefixes) num_plotted += 1 items = [ record.CHROM, record.POS, record.POS + len(trrecord.ref_allele) ] if args.thresh: items.extend(GetThresh(trrecord, samplelists=sample_lists)) if args.afreq: items.extend( GetAFreq(trrecord, samplelists=sample_lists, uselength=args.use_length)) if args.acount: items.extend( GetAFreq(trrecord, samplelists=sample_lists, uselength=args.use_length, count=True)) if args.hwep: items.extend( GetHWEP(trrecord, samplelists=sample_lists, uselength=args.use_length)) if args.het: items.extend( GetHet(trrecord, samplelists=sample_lists, uselength=args.use_length)) if args.mean: items.extend(GetMean(trrecord, samplelists=sample_lists)) if args.mode: items.extend(GetMode(trrecord, samplelists=sample_lists)) if args.var: items.extend(GetVariance(trrecord, samplelists=sample_lists)) if args.numcalled: items.extend(GetNumSamples(trrecord, samplelists=sample_lists)) outf.write("\t".join([str(item) for item in items]) + "\n") outf.close() return 0
def main(): # do an association test of the combined dosage of the listed alleles vs the listed phenotypes in each ethnicity parser = argparse.ArgumentParser() #parser.add_argument('chrom', type=int) #parser.add_argument('pos', type=int) parser.add_argument('var_file') #parser.add_argument('--phenotypes', nargs='+') #parser.add_argument('--alleles', type=int, nargs='+') #allele indicies, i.e. ths SNP is present in alleles 0 (ref), 2 (2nd alt) and 5 args = parser.parse_args() print( 'str\tvariant\tethnicity\tvar frequency\tstr var r2\tphenotype\tstr p-val\tvar p-val\tstr p-val conditioning on var' ) scovs = np.load(f'{ukb}/traits/shared_covars/shared_covars.npy') variants = pl.read_csv(args.var_file, sep='\t') for i in range(variants.shape[0]): chrom = variants[i, 'chrom'] pos = variants[i, 'pos'] str_ = f'{chrom}:{pos}' var = next( cyvcf2.VCF( f'{ukb}/str_imputed/runs/first_pass/vcfs/annotated_strs/chr{chrom}.vcf.gz' )(str_)) alleles = [int(num) for num in variants[i, 'alleles'].split(',')] name = variants[i, 'name'] phenotypes = variants[i, 'phenos'].split(',') for phenotype in phenotypes: for ethnicity in ('white_brits', 'black', 'south_asian', 'chinese', 'irish', 'white_other'): #ethnicity total_samp_idx = sample_utils.get_samples_idx_ethnicity( ethnicity) total_var_gts = var_dosage_gts(var, total_samp_idx, alleles) total_str_gts = str_dosage_gts(var, total_samp_idx) var_freq = np.sum(total_var_gts) / (2 * total_var_gts.shape[0]) corr = np.corrcoef(total_var_gts, total_str_gts) assert corr.shape == (2, 2) str_var_r2 = corr[0, 1]**2 #ethnicity,pheno pcovs = np.load( f'{ukb}/traits/subset_transformed_phenotypes/{ethnicity}/{phenotype}.npy' ) samps = sample_utils.get_ordered_samples_phenotype( ethnicity, phenotype).reshape(-1, 1) covs = python_array_utils.merge_arrays( python_array_utils.merge_arrays(samps, pcovs), scovs) outcomes = covs[:, 1] covs = covs[:, 2:] samp_idx = sample_utils.get_samples_idx_phenotype( ethnicity, phenotype) var_gts = standardize(var_dosage_gts(var, samp_idx, alleles)) str_gts = standardize(str_dosage_gts(var, samp_idx)) str_best_guess_gts = trh.HarmonizeRecord( vcfrecord=var, vcftype='beagle-hipstr').GetGenotypeIndicies()[ samp_idx, :-1] str_p = OLS( outcomes, np.hstack((covs, np.ones((covs.shape[0], 1)), str_gts.reshape(-1, 1)))).fit().pvalues[-1] if np.all(var_gts == 0) or np.all(var_gts == 2): var_p = 1 str_cond_p = str_p else: var_p = OLS( outcomes, np.hstack((covs, np.ones((covs.shape[0], 1)), var_gts.reshape(-1, 1)))).fit().pvalues[-1] str_cond_p = OLS( outcomes, np.hstack((covs, np.ones( (covs.shape[0], 1)), var_gts.reshape(-1, 1), str_gts.reshape(-1, 1)))).fit().pvalues[-1] print( f'{str_}\t{name}\t{ethnicity}\t{var_freq:.3g}\t{str_var_r2:.3g}\t{phenotype}\t{str_p:.3g}\t{var_p:.3g}\t{str_cond_p:.3g}' )