Пример #1
0
    def check_concordancies(self, data_list, assays, chipval):
        hwe_values = [0.0] * len(data_list)
        maf_values = [0.0] * len(data_list)
        obs = [0.0] * 3
        exp = [0.0] * 3
        allele_ref_1 = ""
        allele_alt_1 = ""
        allele_ref_2 = ""
        allele_alt_2 = ""
        #print "CHECK_CONC:", len(data_list)
        for i, vcf_record in enumerate(data_list):
            if len(vcf_record) > 0:
                vcfr = VCFrecord(vcf_record)
                probidx = vcfr.get_probidx()
                homref_count, het_count, homalt_count, nc_count, miss_count = self.vcfr.get_allele_counts_from_array(
                    data)
                allele_a, allele_b = vcfr.get_alleles()
                if allele_ref_1 == "":
                    allele_ref_1 = allele_a
                    allele_alt_1 = allele_b
                    # Add 1 to prevent 0-divide
                    obs[0] = homref_count + 1
                    obs[1] = het_count + 1
                    obs[2] = homalt_count + 1
                else:
                    allele_ref_2 = allele_a
                    allele_alt_2 = allele_b
                    exp[0] = homref_count + 1
                    exp[1] = het_count + 1
                    exp[2] = homalt_count + 1
                    if (allele_ref_1 != allele_ref_2) or (allele_alt_1 !=
                                                          allele_alt_2):
                        varid = vcfr.get_varid(data)
                        posn = vcfr.get_posn(data)
                        self.allele_discord_count += 1
                        logging.info(
                            "Allele discordancy: assay1=%s, assay2=%s, varid=%s, posn=%d, ref1=%s, alt1=%s, ref2=%s, alt2=%s",
                            assays[0], assays[i], varid, int(posn),
                            allele_ref_1, allele_alt_1, allele_ref_2,
                            allele_alt_2)
                        #print "Allele discord"
                        return False

                    chi_stat, chi_p_value = chisquare(obs, f_exp=exp)
                    varid = vcfr.get_varid(data)
                    posn = vcfr.get_posn(data)
                    if chi_p_value < chipval:
                        self.chisq_count += 1
                        logging.info(
                            "CHI SQ test REJECT: assay1=%s, assay2=%s, varid=%s, posn=%d, chistat=%f, p_val=%e, chipval=%e, obs=%s, exp=%s, at %d",
                            assays[0], assays[i], varid, int(posn), chi_stat,
                            chi_p_value, chipval, str(obs), str(exp), i)
                        #print "CHISQ discord"
                        return False
                    logging.info(
                        "CHI SQ test OK: assay1=%s, assay2=%s, varid=%s, posn=%d, chistat=%f, p_val=%e, obs=%s, exp=%s, at %d",
                        assays[0], assays[i], varid, int(posn), chi_stat,
                        chi_p_value, str(obs), str(exp), i)

        return True
Пример #2
0
  def process_variant_detail_vcf(self, record, assaytype):
    """Process info file variant detail records
       Set up a json-stype document and add it to the
      variant buffer
    """
    doc = {}
    doc["assaytype"] = assaytype
    vcfr = VCFrecord(record)
    prfx, sfx = vcfr.get_prfx_sfx()
    doc["rsid"] = vcfr.get_varid()
    # always store chromosome as a 2-digit string
    doc["chromosome"] = "%.2d" % (int(vcfr.get_chr()))
    alleleA, alleleB = vcfr.get_alleles()
    doc["alleleA"] = alleleA
    doc["alleleB"] = alleleB
    doc["position"] = vcfr.get_posn_as_int()
    try:
      doc["ref_maf"] = float(vcfr.get_info_value("RefPanelAF"))
    except:
      pass
    try:
      doc["info"] = float(vcfr.get_info_value("INFO"))
    except:
      doc["info"] = 1.0

    self.variantbuff.append(doc)
Пример #3
0
def get_dbsnp_rsid(dbsnpfile, chrom, posn):
  dbsnprec = dbsnpfile.get_dbsnp_file_record(options.dbsnpfile, chrom, int(posn))
  rsid = ""
  refallele = ""
  if dbsnprec != None:
    dbvcf = VCFrecord(dbsnprec)
    rsid = dbvcf.get_varid()
    refallele, altallele = dbvcf.get_alleles()
  return rsid, refallele
Пример #4
0
    def get_combined_array(self,
                           buffer_list,
                           cr_list,
                           assay_list,
                           threshold=0.9):
        """
    For each list of data, for each element of list of data:
    1) Find the col header from the corresonding file_position element
    2) Use the col_header to find the combined postion
    3) Place the data_element in the combined postion *
    TODO - conflict resolution, what to do if a slot is already occupied
    TODO - CR check
    """
        #print "COMBO", self.combined_positions
        #
        #print "ASSAY_LIST: %s" % (str(assay_list))
        assay_posns = {}

        for i, assaytype in enumerate(assay_list):
            assay_posns[i] = assaytype

        #print "ASSAY_POSNS: %s" % (str(assay_posns))

        combo_array = ["."] * len(self.combined_positions)
        #print "BUFFL", len(buffer_list)
        for i, vcf_record in enumerate(buffer_list):
            if len(vcf_record) > 0:
                #print "asstp: %d, %s" % (i, assay_list[i])
                vcfr = VCFrecord(vcf_record)
                prfx, data_list = vcfr.get_prfx_sfx()
                probidx = vcfr.get_probidx()
                rsid = vcfr.get_varid()
                hasAT = vcfr.has_fmt("AT")
                for j, dataelem in enumerate(data_list):
                    if data_list[j] != ".":
                        cpos = self.combined_positions[self.file_positions[i]
                                                       [j]]
                        geno = self.call_geno_for_threshold(
                            data_list[j], probidx, threshold)
                        if (hasAT == False):
                            geno = geno + ":" + self.assay_abbrev[
                                assay_list[i]]
                        if combo_array[cpos] != ".":
                            self.geno_overlap_count += 1
                            #print "OVERLAP %s:%s - %s vs %s" % (rsid, self.file_positions[i][j], combo_array[cpos], geno)
                            geno = self.call_genotype(combo_array[cpos], geno,
                                                      probidx)
                        combo_array[cpos] = geno

        return combo_array
Пример #5
0
def main(options):
  #included_assaytypes = {"biggertest":1, "bigtest":1, "affy":1, "illumina":1, "broad":1, "metabo":1, "exome":1}
  #included_assaytypes = {"bigtest":1, "affy":1, "illumina":1, "broad":1, "metabo":1, "exome":1}
  #included_assaytypes = {"affy":1, "illumina":1, "broad":1, "metabo":1, "exome":1}
  #included_assaytypes = {"affy":1, "illumina":1, "broad":1, "exome":1}
  #included_assaytypes = {"affy":1, "illumina":1, "broad":1}
  #included_assaytypes = {"affy":1, "illumina":1}
  included_assaytypes = {"broad":1}
  #included_assaytypes = {"metabo":1}
  #included_assaytypes = {"affy":1}
  #included_assaytypes = {"bigtest":1}
  #included_assaytypes = {"biggertest":1}
  rsids = []
  godb = GoDb()

  try:
    if options.snpfile != None:
      fh = open(options.snpfile, "r") 
      rsids = load_snpfile_data(fh)
    else:
      rsids = options.rsids.split(",")
  except IOError as e:
    print "I/O error({0}): {1}".format(e.errno, e.strerror)
    exit()
  except TypeError as e:
    print "Missing arguments ", e
    exit()
  except:
    logging.info("Unexpected error: %s", str(sys.exc_info()))
    sys.exit()

# Step 0 - initialise db connection and instanciate helper objects
  mafh = Mafhelper()
  hweh = Hwehelper()
# Data structures
  atype_list = []
  atype_posns = {}
  marker_list = []
  rsid_assaytypes = {}
  rsid_dict = {}
  rsid_prfx_dict = {}
  rsid_cr_dict = {}
  rsid_info_dict = {}
  hdr_pref = ["#CHROM",  "POS", "ID",  "REF", "ALT", "QUAL",  "FILTER",  "INFO",  "FORMAT"]

# Step 1 - get the list of entries for each rsid - one per assaytype

  for rsid in rsids:
    #logging.info("Processing rsid = %s", rsid)
    docs = godb.get_multiple_variants(rsid)
    if docs.count() > 0:
      rsid_assaytypes[rsid] = []
    else:
      logging.info("RSID %s NOTFOUND", rsid)
  #print docs

  # Step 1a - collect assaytypes and marker documents
  # At this point we're establishing a list order which must be observed throughout.
    for doc in docs:
      #logging.info("%s", str(doc))
      if doc["assaytype"] not in included_assaytypes:
        continue
      if doc["assaytype"] not in atype_list:
        atype_list.append(doc["assaytype"])
      rsid_assaytypes[rsid].append(doc)
  logging.info(str(atype_list))
# Step 2 - collect lists of prochis (sample ids) by assaytype
  prochi_list = [[]] * len(atype_list)
  for i, atype in enumerate(atype_list):
    atype_posns[atype] = i
    prochi_list[i] = godb.get_samples(atype)
    #logging.info("SAMP %d, %s, %s", i, atype, str(prochi_list[i]))

  mm = Multibuffermerge(prochi_list)

# Step 3 - get combined col_header positions
# combo is a dict {posn:colname}
  combo = mm.get_combined_positions()
  #print len(combo)
# combocol is a list [colname1, colname2, ..., colname] again we keep the order of this intact
  combocol = mm.get_combined_columns()
  
# Step 4 - for each variant by rsid
  for rsid in rsid_assaytypes:
    if rsid not in rsid_dict:
      rsid_prfx_dict[rsid] = [[]] * len(atype_list)
      rsid_dict[rsid] = [[]] * len(atype_list)
      rsid_cr_dict[rsid] = [[]] * len(atype_list)
      rsid_info_dict[rsid] = [[]] * len(atype_list)
    #print len(rsid_assaytypes[rsid])
    for doc in rsid_assaytypes[rsid]:
      if options.prfx != None:
        fpath = godb.get_full_filepath(doc["assaytype"], doc["chromosome"], options.prfx)
      else:
        fpath = godb.get_filepath(doc["assaytype"], doc["chromosome"])
      logging.info("Assaytype=%s fpath=%s", doc["assaytype"], fpath)

      result = godb.get_variant_file_data(fpath, doc["chromosome"], doc["position"])
      if result != None:
        vcfr = VCFrecord(result)
        varid = vcfr.get_varid()
        if varid == rsid:
          rec = result
          maf, ma, cr = mafh.get_maf_and_cr(vcfr)
          # TODO - ALSO check maf, also apply QC filter at individual record level
          rsid_cr_dict[doc["rsid"]][atype_posns[doc["assaytype"]]] = cr
          rsid_dict[doc["rsid"]][atype_posns[doc["assaytype"]]] = rec
          logging.info("%s (%s), maf=%s, ma=%s, cr=%s" % (doc["rsid"], doc["assaytype"], maf, ma, cr))
  
  #print combocol
# Step 5 - execute the merge process
  print "\t".join(hdr_pref + combocol)
  count = 0
  concordant = True
  for rsid in rsid_dict:
    if len(rsid_dict[rsid][0]) > 0:
      if options.check == 'Y':
        concordant = mm.check_concordancies(rsid_dict[rsid], atype_list, options.chipval)

      if concordant == True:
        comborec = mm.get_combined_array(rsid_dict[rsid], rsid_cr_dict[rsid], atype_list)
        vcfr = VCFrecord(rsid_dict[rsid][0])
        prfx,sfx = vcfr.get_prfx_sfx()
        if len(prfx) > 0:
          logging.info("PRFX = %s, for %s", str(prfx), rsid)
          prfx[8] += ":AT"
          outrec = prfx + comborec
          print "\t".join(outrec)
          count += 1
        else:
          logging.info("RSID %s NOTFOUND (2)", rsid)
          pass
      else:
        logging.info("Concordancy check fail for - %s" % (rsid))

  #chi_test_count, allele_disc_count, overlap_count, cr_check_count = mm.get_counts()
  #logging.info("Overlap check count = %d, cr_check_count = %d", overlap_count, cr_check_count)
  chi_test_count, allele_disc_count, overlap_count = mm.get_counts()
  logging.info("CHI test count = %d, Allele discord count = %d, Overlap check count = %d", 
    chi_test_count, allele_disc_count, overlap_count)

  return count 
def main(options):
    hdrData = ["id"]
    sampleDict = {}
    colPosns = {}
    RefAlleleDict = {}
    AltAlleleDict = {}
    count = 0

    mafh = Mafhelper()

    for line in sys.stdin:
        line = line.strip()
        if (line.startswith('##')):
            pass
        else:
            vcfr = VCFrecord(line)
            prfx, sfx = vcfr.get_prfx_sfx()
            #print prfx
            if (line.startswith('#')):
                # Parse out the header record.
                for i, col_hdr in enumerate(sfx):
                    colPosns[i] = col_hdr
                    sampleDict[col_hdr] = []
            else:
                flip = False
                varid = vcfr.get_varid()
                #logging.info("varid=%s", varid)
                ref, alt = vcfr.get_alleles()
                probidx = vcfr.get_probidx()
                hdr_allele = alt
                homref_count, het_count, homalt_count, nc_count, miss_count = vcfr.get_allele_counts(
                )
                call_count = homref_count + het_count + homalt_count
                maf, ma = mafh.maf(het_count, homref_count, ref, homalt_count,
                                   alt, nc_count)
                RefAlleleDict[varid] = ref
                AltAlleleDict[varid] = alt
                #if ma == ref:
                #  flip = True
                #  hdr_allele = ref
                #  logging.info("FLIP for %s, %s, %s", varid, ref, alt)
                hdrData.extend(
                    (varid, varid + "_prob", varid + "_plat", varid + "_min"))
                for i, str_geno in enumerate(sfx):
                    if str_geno != ".":
                        geno = str_geno.split(":")
                        max_prob, max_idx = get_max_prob(geno, probidx)
                        i_call = icalls[geno[0]]
                        if flip == True:
                            if i_call == "0":
                                i_call == "2"
                            elif i_call == "2":
                                i_call = "0"
                        if options.platform != None:
                            sampleDict[colPosns[i]].extend(
                                (str(i_call), str(max_prob), options.platform,
                                 hdr_allele))
                        else:
                            sampleDict[colPosns[i]].extend(
                                (str(i_call), str(max_prob),
                                 assay_expand[geno[probidx + 1]], hdr_allele))
                    else:
                        sampleDict[colPosns[i]].extend(("", "", "", ""))

    print ",".join(hdrData)
    for samp in sampleDict:
        count += 1
        print ",".join([samp] + sampleDict[samp])
    return count