def check_concordancies(self, data_list, assays, chipval): hwe_values = [0.0] * len(data_list) maf_values = [0.0] * len(data_list) obs = [0.0] * 3 exp = [0.0] * 3 allele_ref_1 = "" allele_alt_1 = "" allele_ref_2 = "" allele_alt_2 = "" #print "CHECK_CONC:", len(data_list) for i, vcf_record in enumerate(data_list): if len(vcf_record) > 0: vcfr = VCFrecord(vcf_record) probidx = vcfr.get_probidx() homref_count, het_count, homalt_count, nc_count, miss_count = self.vcfr.get_allele_counts_from_array( data) allele_a, allele_b = vcfr.get_alleles() if allele_ref_1 == "": allele_ref_1 = allele_a allele_alt_1 = allele_b # Add 1 to prevent 0-divide obs[0] = homref_count + 1 obs[1] = het_count + 1 obs[2] = homalt_count + 1 else: allele_ref_2 = allele_a allele_alt_2 = allele_b exp[0] = homref_count + 1 exp[1] = het_count + 1 exp[2] = homalt_count + 1 if (allele_ref_1 != allele_ref_2) or (allele_alt_1 != allele_alt_2): varid = vcfr.get_varid(data) posn = vcfr.get_posn(data) self.allele_discord_count += 1 logging.info( "Allele discordancy: assay1=%s, assay2=%s, varid=%s, posn=%d, ref1=%s, alt1=%s, ref2=%s, alt2=%s", assays[0], assays[i], varid, int(posn), allele_ref_1, allele_alt_1, allele_ref_2, allele_alt_2) #print "Allele discord" return False chi_stat, chi_p_value = chisquare(obs, f_exp=exp) varid = vcfr.get_varid(data) posn = vcfr.get_posn(data) if chi_p_value < chipval: self.chisq_count += 1 logging.info( "CHI SQ test REJECT: assay1=%s, assay2=%s, varid=%s, posn=%d, chistat=%f, p_val=%e, chipval=%e, obs=%s, exp=%s, at %d", assays[0], assays[i], varid, int(posn), chi_stat, chi_p_value, chipval, str(obs), str(exp), i) #print "CHISQ discord" return False logging.info( "CHI SQ test OK: assay1=%s, assay2=%s, varid=%s, posn=%d, chistat=%f, p_val=%e, obs=%s, exp=%s, at %d", assays[0], assays[i], varid, int(posn), chi_stat, chi_p_value, str(obs), str(exp), i) return True
def process_variant_detail_vcf(self, record, assaytype): """Process info file variant detail records Set up a json-stype document and add it to the variant buffer """ doc = {} doc["assaytype"] = assaytype vcfr = VCFrecord(record) prfx, sfx = vcfr.get_prfx_sfx() doc["rsid"] = vcfr.get_varid() # always store chromosome as a 2-digit string doc["chromosome"] = "%.2d" % (int(vcfr.get_chr())) alleleA, alleleB = vcfr.get_alleles() doc["alleleA"] = alleleA doc["alleleB"] = alleleB doc["position"] = vcfr.get_posn_as_int() try: doc["ref_maf"] = float(vcfr.get_info_value("RefPanelAF")) except: pass try: doc["info"] = float(vcfr.get_info_value("INFO")) except: doc["info"] = 1.0 self.variantbuff.append(doc)
def get_dbsnp_rsid(dbsnpfile, chrom, posn): dbsnprec = dbsnpfile.get_dbsnp_file_record(options.dbsnpfile, chrom, int(posn)) rsid = "" refallele = "" if dbsnprec != None: dbvcf = VCFrecord(dbsnprec) rsid = dbvcf.get_varid() refallele, altallele = dbvcf.get_alleles() return rsid, refallele
def get_combined_array(self, buffer_list, cr_list, assay_list, threshold=0.9): """ For each list of data, for each element of list of data: 1) Find the col header from the corresonding file_position element 2) Use the col_header to find the combined postion 3) Place the data_element in the combined postion * TODO - conflict resolution, what to do if a slot is already occupied TODO - CR check """ #print "COMBO", self.combined_positions # #print "ASSAY_LIST: %s" % (str(assay_list)) assay_posns = {} for i, assaytype in enumerate(assay_list): assay_posns[i] = assaytype #print "ASSAY_POSNS: %s" % (str(assay_posns)) combo_array = ["."] * len(self.combined_positions) #print "BUFFL", len(buffer_list) for i, vcf_record in enumerate(buffer_list): if len(vcf_record) > 0: #print "asstp: %d, %s" % (i, assay_list[i]) vcfr = VCFrecord(vcf_record) prfx, data_list = vcfr.get_prfx_sfx() probidx = vcfr.get_probidx() rsid = vcfr.get_varid() hasAT = vcfr.has_fmt("AT") for j, dataelem in enumerate(data_list): if data_list[j] != ".": cpos = self.combined_positions[self.file_positions[i] [j]] geno = self.call_geno_for_threshold( data_list[j], probidx, threshold) if (hasAT == False): geno = geno + ":" + self.assay_abbrev[ assay_list[i]] if combo_array[cpos] != ".": self.geno_overlap_count += 1 #print "OVERLAP %s:%s - %s vs %s" % (rsid, self.file_positions[i][j], combo_array[cpos], geno) geno = self.call_genotype(combo_array[cpos], geno, probidx) combo_array[cpos] = geno return combo_array
def main(options): #included_assaytypes = {"biggertest":1, "bigtest":1, "affy":1, "illumina":1, "broad":1, "metabo":1, "exome":1} #included_assaytypes = {"bigtest":1, "affy":1, "illumina":1, "broad":1, "metabo":1, "exome":1} #included_assaytypes = {"affy":1, "illumina":1, "broad":1, "metabo":1, "exome":1} #included_assaytypes = {"affy":1, "illumina":1, "broad":1, "exome":1} #included_assaytypes = {"affy":1, "illumina":1, "broad":1} #included_assaytypes = {"affy":1, "illumina":1} included_assaytypes = {"broad":1} #included_assaytypes = {"metabo":1} #included_assaytypes = {"affy":1} #included_assaytypes = {"bigtest":1} #included_assaytypes = {"biggertest":1} rsids = [] godb = GoDb() try: if options.snpfile != None: fh = open(options.snpfile, "r") rsids = load_snpfile_data(fh) else: rsids = options.rsids.split(",") except IOError as e: print "I/O error({0}): {1}".format(e.errno, e.strerror) exit() except TypeError as e: print "Missing arguments ", e exit() except: logging.info("Unexpected error: %s", str(sys.exc_info())) sys.exit() # Step 0 - initialise db connection and instanciate helper objects mafh = Mafhelper() hweh = Hwehelper() # Data structures atype_list = [] atype_posns = {} marker_list = [] rsid_assaytypes = {} rsid_dict = {} rsid_prfx_dict = {} rsid_cr_dict = {} rsid_info_dict = {} hdr_pref = ["#CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO", "FORMAT"] # Step 1 - get the list of entries for each rsid - one per assaytype for rsid in rsids: #logging.info("Processing rsid = %s", rsid) docs = godb.get_multiple_variants(rsid) if docs.count() > 0: rsid_assaytypes[rsid] = [] else: logging.info("RSID %s NOTFOUND", rsid) #print docs # Step 1a - collect assaytypes and marker documents # At this point we're establishing a list order which must be observed throughout. for doc in docs: #logging.info("%s", str(doc)) if doc["assaytype"] not in included_assaytypes: continue if doc["assaytype"] not in atype_list: atype_list.append(doc["assaytype"]) rsid_assaytypes[rsid].append(doc) logging.info(str(atype_list)) # Step 2 - collect lists of prochis (sample ids) by assaytype prochi_list = [[]] * len(atype_list) for i, atype in enumerate(atype_list): atype_posns[atype] = i prochi_list[i] = godb.get_samples(atype) #logging.info("SAMP %d, %s, %s", i, atype, str(prochi_list[i])) mm = Multibuffermerge(prochi_list) # Step 3 - get combined col_header positions # combo is a dict {posn:colname} combo = mm.get_combined_positions() #print len(combo) # combocol is a list [colname1, colname2, ..., colname] again we keep the order of this intact combocol = mm.get_combined_columns() # Step 4 - for each variant by rsid for rsid in rsid_assaytypes: if rsid not in rsid_dict: rsid_prfx_dict[rsid] = [[]] * len(atype_list) rsid_dict[rsid] = [[]] * len(atype_list) rsid_cr_dict[rsid] = [[]] * len(atype_list) rsid_info_dict[rsid] = [[]] * len(atype_list) #print len(rsid_assaytypes[rsid]) for doc in rsid_assaytypes[rsid]: if options.prfx != None: fpath = godb.get_full_filepath(doc["assaytype"], doc["chromosome"], options.prfx) else: fpath = godb.get_filepath(doc["assaytype"], doc["chromosome"]) logging.info("Assaytype=%s fpath=%s", doc["assaytype"], fpath) result = godb.get_variant_file_data(fpath, doc["chromosome"], doc["position"]) if result != None: vcfr = VCFrecord(result) varid = vcfr.get_varid() if varid == rsid: rec = result maf, ma, cr = mafh.get_maf_and_cr(vcfr) # TODO - ALSO check maf, also apply QC filter at individual record level rsid_cr_dict[doc["rsid"]][atype_posns[doc["assaytype"]]] = cr rsid_dict[doc["rsid"]][atype_posns[doc["assaytype"]]] = rec logging.info("%s (%s), maf=%s, ma=%s, cr=%s" % (doc["rsid"], doc["assaytype"], maf, ma, cr)) #print combocol # Step 5 - execute the merge process print "\t".join(hdr_pref + combocol) count = 0 concordant = True for rsid in rsid_dict: if len(rsid_dict[rsid][0]) > 0: if options.check == 'Y': concordant = mm.check_concordancies(rsid_dict[rsid], atype_list, options.chipval) if concordant == True: comborec = mm.get_combined_array(rsid_dict[rsid], rsid_cr_dict[rsid], atype_list) vcfr = VCFrecord(rsid_dict[rsid][0]) prfx,sfx = vcfr.get_prfx_sfx() if len(prfx) > 0: logging.info("PRFX = %s, for %s", str(prfx), rsid) prfx[8] += ":AT" outrec = prfx + comborec print "\t".join(outrec) count += 1 else: logging.info("RSID %s NOTFOUND (2)", rsid) pass else: logging.info("Concordancy check fail for - %s" % (rsid)) #chi_test_count, allele_disc_count, overlap_count, cr_check_count = mm.get_counts() #logging.info("Overlap check count = %d, cr_check_count = %d", overlap_count, cr_check_count) chi_test_count, allele_disc_count, overlap_count = mm.get_counts() logging.info("CHI test count = %d, Allele discord count = %d, Overlap check count = %d", chi_test_count, allele_disc_count, overlap_count) return count
def main(options): hdrData = ["id"] sampleDict = {} colPosns = {} RefAlleleDict = {} AltAlleleDict = {} count = 0 mafh = Mafhelper() for line in sys.stdin: line = line.strip() if (line.startswith('##')): pass else: vcfr = VCFrecord(line) prfx, sfx = vcfr.get_prfx_sfx() #print prfx if (line.startswith('#')): # Parse out the header record. for i, col_hdr in enumerate(sfx): colPosns[i] = col_hdr sampleDict[col_hdr] = [] else: flip = False varid = vcfr.get_varid() #logging.info("varid=%s", varid) ref, alt = vcfr.get_alleles() probidx = vcfr.get_probidx() hdr_allele = alt homref_count, het_count, homalt_count, nc_count, miss_count = vcfr.get_allele_counts( ) call_count = homref_count + het_count + homalt_count maf, ma = mafh.maf(het_count, homref_count, ref, homalt_count, alt, nc_count) RefAlleleDict[varid] = ref AltAlleleDict[varid] = alt #if ma == ref: # flip = True # hdr_allele = ref # logging.info("FLIP for %s, %s, %s", varid, ref, alt) hdrData.extend( (varid, varid + "_prob", varid + "_plat", varid + "_min")) for i, str_geno in enumerate(sfx): if str_geno != ".": geno = str_geno.split(":") max_prob, max_idx = get_max_prob(geno, probidx) i_call = icalls[geno[0]] if flip == True: if i_call == "0": i_call == "2" elif i_call == "2": i_call = "0" if options.platform != None: sampleDict[colPosns[i]].extend( (str(i_call), str(max_prob), options.platform, hdr_allele)) else: sampleDict[colPosns[i]].extend( (str(i_call), str(max_prob), assay_expand[geno[probidx + 1]], hdr_allele)) else: sampleDict[colPosns[i]].extend(("", "", "", "")) print ",".join(hdrData) for samp in sampleDict: count += 1 print ",".join([samp] + sampleDict[samp]) return count