def extract_msa(base_dname, base_fname, locus_list, min_freq, verbose): # Download human genome and HISAT2 index HISAT2_fnames = ["grch38", "genome.fa", "genome.fa.fai"] if not typing_common.check_files(HISAT2_fnames): typing_common.download_genome_and_index(ex_path) # Load allele frequency information allele_freq = {} if min_freq > 0.0: excel = openpyxl.load_workbook( "hisatgenotype_db/CODIS/NIST-US1036-AlleleFrequencies.xlsx") sheet = excel.get_sheet_by_name(u'All data, n=1036') for col in range(2, 100): locus_name = sheet.cell(row=3, column=col).value if not locus_name: break locus_name = locus_name.encode('ascii', 'ignore') locus_name = locus_name.upper() assert locus_name not in allele_freq allele_freq[locus_name] = {} for row in range(4, 101): allele_id = sheet.cell(row=row, column=1).value allele_id = str(allele_id) freq = sheet.cell(row=row, column=col).value if not freq: continue allele_freq[locus_name][allele_id] = float(freq) excel.close() CODIS_seq = orig_CODIS_seq if len(locus_list) > 0: new_CODIS_seq = {} for locus_name, fields in CODIS_seq.items(): if locus_name in locus_list: new_CODIS_seq[locus_name] = fields CODIS_seq = new_CODIS_seq # Add some additional sequences to allele sequences to make them reasonably long for typing and assembly for locus_name, fields in CODIS_seq.items(): _, left_seq, repeat_seq, right_seq = fields allele_seq = left_seq + repeat_seq + right_seq left_flank_seq, right_flank_seq = get_flanking_seqs(allele_seq) CODIS_seq[locus_name][1] = left_flank_seq + left_seq CODIS_seq[locus_name][3] = right_seq + right_flank_seq print >> sys.stderr, "%s is found on the reference genome (GRCh38)" % locus_name for locus_name in CODIS_seq.keys(): alleles = [] for line in open("hisatgenotype_db/CODIS/codis.dat"): locus_name2, allele_id, repeat_st = line.strip().split('\t') if locus_name != locus_name2: continue if min_freq > 0.0: assert locus_name in allele_freq if allele_id not in allele_freq[locus_name] or \ allele_freq[locus_name][allele_id] < min_freq: continue alleles.append([allele_id, repeat_st]) # From [TTTC]3TTTTTTCT[CTTT]20CTCC[TTCC]2 # To [['TTTC', [3]], ['TTTTTTCT', [1]], ['CTTT', [20]], ['CTCC', [1]], ['TTCC', [2]]] def read_allele(repeat_st): allele = [] s = 0 while s < len(repeat_st): ch = repeat_st[s] if ch == ' ': s += 1 continue assert ch in "[ACGT" if ch == '[': s += 1 repeat = "" while s < len(repeat_st): nt = repeat_st[s] if nt in "ACGT": repeat += nt s += 1 else: assert nt == ']' s += 1 break assert s < len(repeat_st) num = 0 while s < len(repeat_st): digit = repeat_st[s] if digit.isdigit(): num = num * 10 + int(digit) s += 1 else: break assert num > 0 allele.append([set([repeat]), set([num])]) else: repeat = "" while s < len(repeat_st): nt = repeat_st[s] if nt in "ACGT": repeat += nt s += 1 else: assert nt == ' ' or nt == '[' break allele.append([set([repeat]), set([1])]) # Sanity check cmp_repeat_st = "" for repeats, repeat_nums in allele: repeat = list(repeats)[0] repeat_num = list(repeat_nums)[0] if repeat_num > 1 or locus_name == "D8S1179": cmp_repeat_st += "[" cmp_repeat_st += repeat if repeat_num > 1 or locus_name == "D8S1179": cmp_repeat_st += "]%d" % repeat_num assert repeat_st.replace(' ', '') == cmp_repeat_st.replace(' ', '') return allele alleles = [[allele_id, read_allele(repeat_st)] for allele_id, repeat_st in alleles] def to_sequence(repeat_st): sequence = "" for repeats, repeat_nums in repeat_st: repeat = list(repeats)[0] repeat_num = list(repeat_nums)[0] sequence += (repeat * repeat_num) return sequence def remove_redundant_alleles(alleles): seq_to_ids = {} new_alleles = [] for allele_id, repeat_st in alleles: allele_seq = to_sequence(repeat_st) if allele_seq in seq_to_ids: print >> sys.stderr, "Warning) %s: %s has the same sequence as %s" % \ (locus_name, allele_id, seq_to_ids[allele_seq]) continue if allele_seq not in seq_to_ids: seq_to_ids[allele_seq] = [allele_id] else: seq_to_ids[allele_seq].append(allele_id) new_alleles.append([allele_id, repeat_st]) return new_alleles alleles = remove_redundant_alleles(alleles) allele_seqs = [[allele_id, to_sequence(repeat_st)] for allele_id, repeat_st in alleles] ref_allele_st, ref_allele_left, ref_allele, ref_allele_right = CODIS_seq[ locus_name] ref_allele_st = read_allele(ref_allele_st) for allele_id, allele_seq in allele_seqs: if ref_allele == allele_seq: CODIS_ref_name[locus_name] = allele_id break # Add GRCh38 allele if locus_name not in CODIS_ref_name: allele_id = "GRCh38" CODIS_ref_name[locus_name] = allele_id allele_seqs = [[allele_id, ref_allele]] + allele_seqs alleles = [[allele_id, ref_allele_st]] + alleles print >> sys.stderr, "%s: %d alleles with reference allele as %s" % ( locus_name, len(alleles), CODIS_ref_name[locus_name]) if verbose: print >> sys.stderr, "\t", ref_allele_left, ref_allele, ref_allele_right for allele_id, allele in alleles: print >> sys.stderr, allele_id, "\t", allele # Create a backbone sequence assert len(alleles) > 0 backbone_allele = deepcopy(alleles[-1][1]) for allele_id, allele_st in reversed(alleles[:-1]): if verbose: print >> sys.stderr print >> sys.stderr, allele_id print >> sys.stderr, "backbone :", backbone_allele print >> sys.stderr, "allele :", allele_st backbone_allele = combine_alleles(backbone_allele, allele_st) msf_allele_seq, msf_backbone_seq = msf_alignment( backbone_allele, allele_st) if verbose: print >> sys.stderr, "combined backbone:", backbone_allele print >> sys.stderr, "msf_allele_seq :", msf_allele_seq print >> sys.stderr, "msf_backbone_seq:", msf_backbone_seq print >> sys.stderr allele_dic = {} for allele_id, allele_seq in allele_seqs: allele_dic[allele_id] = allele_seq allele_repeat_msf = {} for allele_id, allele_st in alleles: msf_allele_seq, msf_backbone_seq = msf_alignment( backbone_allele, allele_st) allele_repeat_msf[allele_id] = msf_allele_seq # Sanity check assert len(allele_dic) == len(allele_repeat_msf) repeat_len = None for allele_id, repeat_msf in allele_repeat_msf.items(): if not repeat_len: repeat_len = len(repeat_msf) else: assert repeat_len == len(repeat_msf) # Creat full multiple sequence alignment ref_allele_id = CODIS_ref_name[locus_name] allele_msf = {} for allele_id, repeat_msf in allele_repeat_msf.items(): allele_msf[ allele_id] = ref_allele_left + repeat_msf + ref_allele_right # Make sure the length of allele ID is short, less than 20 characters max_allele_id_len = max( [len(allele_id) for allele_id in allele_dic.keys()]) assert max_allele_id_len < 20 # Write MSF (multiple sequence alignment file) msf_len = len(ref_allele_left) + len(ref_allele_right) + repeat_len msf_fname = "%s_gen.msf" % locus_name msf_file = open(msf_fname, 'w') for s in range(0, msf_len, 50): for allele_id, msf in allele_msf.items(): assert len(msf) == msf_len allele_name = "%s*%s" % (locus_name, allele_id) print >> msf_file, "%20s" % allele_name, for s2 in range(s, min(msf_len, s + 50), 10): print >> msf_file, " %s" % msf[s2:s2 + 10], print >> msf_file if s + 50 >= msf_len: break print >> msf_file msf_file.close() # Write FASTA file fasta_fname = "%s_gen.fasta" % locus_name fasta_file = open(fasta_fname, 'w') for allele_id, allele_seq in allele_seqs: gen_seq = ref_allele_left + allele_seq + ref_allele_right print >> fasta_file, ">%s*%s %d bp" % (locus_name, allele_id, len(gen_seq)) for s in range(0, len(gen_seq), 60): print >> fasta_file, gen_seq[s:s + 60] fasta_file.close()
def build_genotype_genome(base_fname, inter_gap, intra_gap, threads, database_list, use_clinvar, use_commonvar, aligner, graph_index, verbose): # Download HISAT2 index typing_common.download_genome_and_index() # Load genomic sequences chr_dic, chr_names, chr_full_names = typing_common.read_genome("genome.fa") genotype_vars = {} genotype_haplotypes = {} genotype_clnsig = {} if use_clinvar: # Extract variants from the ClinVar database CLINVAR_fnames = [ "clinvar.vcf.gz", "clinvar.snp", "clinvar.haplotype", "clinvar.clnsig" ] if not typing_common.check_files(CLINVAR_fnames): if not os.path.exists("clinvar.vcf.gz"): os.system("wget ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/"\ "vcf_GRCh38/archive/2017/clinvar_20170404.vcf.gz") assert os.path.exists("clinvar.vcf.gz") extract_cmd = ["hisat2_extract_snps_haplotypes_VCF.py"] extract_cmd += [ "--inter-gap", str(inter_gap), "--intra-gap", str(intra_gap), "--genotype-vcf", "clinvar.vcf.gz", "genome.fa", "/dev/null", "clinvar" ] if verbose: print("\tRunning:", ' '.join(extract_cmd), file=sys.stderr) proc = subprocess.Popen(extract_cmd, stdout=open("/dev/null", 'w'), stderr=open("/dev/null", 'w')) proc.communicate() if not typing_common.check_files(CLINVAR_fnames): print("Error: extract variants from clinvar failed!", file=sys.stderr) sys.exit(1) # Read variants to be genotyped genotype_vars = typing_common.read_variants("clinvar.snp") # Read haplotypes genotype_haplotypes = typing_common.read_haplotypes( "clinvar.haplotype") # Read information about clinical significance genotype_clnsig = read_clnsig("clinvar.clnsig") if use_commonvar: # Extract variants from dbSNP database # TODO: CB Write script to make local uptodate SNP database from dbSNP # ftp://ftp.ncbi.nlm.nih.gov/snp/database/README.create_local_dbSNP.txt commonvar_fbase = "snp144Common" commonvar_fnames = [ "%s.snp" % commonvar_fbase, "%s.haplotype" % commonvar_fbase ] if not typing_common.check_files(commonvar_fnames): if not os.path.exists("%s.txt.gz" % commonvar_fbase): os.system("wget http://hgdownload.cse.ucsc.edu/goldenPath/hg38/"\ "database/%s.txt.gz" % commonvar_fbase) assert os.path.exists("%s.txt.gz" % commonvar_fbase) os.system("gzip -cd %s.txt.gz "\ "| awk 'BEGIN{OFS=\"\t\"} "\ "{if($2 ~ /^chr/) {$2 = substr($2, 4)}; "\ "if($2 == \"M\") {$2 = \"MT\"} print}' > %s.txt" \ % (commonvar_fbase, commonvar_fbase)) extract_cmd = [ "hisat2_extract_snps_haplotypes_UCSC.py", "--inter-gap", str(inter_gap), "--intra-gap", str(intra_gap), "genome.fa", "%s.txt" % commonvar_fbase, commonvar_fbase ] if verbose: print("\tRunning:", ' '.join(extract_cmd), file=sys.stderr) proc = subprocess.Popen(extract_cmd, stdout=open("/dev/null", 'w'), stderr=open("/dev/null", 'w')) proc.communicate() if not typing_common.check_files(commonvar_fnames): print("Error: extract variants from clinvar failed!", file=sys.stderr) sys.exit(1) # Read variants to be genotyped genotype_vars = typing_common.read_variants(commonvar_fnames[0]) # Read haplotypes genotype_haplotypes = typing_common.read_haplotypes( commonvar_fnames[1]) # Genes to be genotyped genotype_genes = {} # Read genes or genomics regions for database_name in database_list: # Extract HLA variants, backbone sequence, and other sequeces typing_common.extract_database_if_not_exists( database_name, [], # locus_list inter_gap, intra_gap, True, # partial? verbose) locus_fname = "%s.locus" % database_name assert os.path.exists(locus_fname) for line in open(locus_fname): locus_name, \ chr, \ left, \ right, \ length, \ exon_str, \ strand \ = line.strip().split() left = int(left) right = int(right) length = int(length) if chr not in chr_names: continue if chr not in genotype_genes: genotype_genes[chr] = [] genotype_genes[chr].append([ left, right, length, locus_name, database_name, exon_str, strand ]) # Write genotype genome var_num = 0 haplotype_num = 0 genome_out_file = open("%s.fa" % base_fname, 'w') locus_out_file = open("%s.locus" % base_fname, 'w') var_out_file = open("%s.snp" % base_fname, 'w') index_var_out_file = open("%s.index.snp" % base_fname, 'w') haplotype_out_file = open("%s.haplotype" % base_fname, 'w') link_out_file = open("%s.link" % base_fname, 'w') coord_out_file = open("%s.coord" % base_fname, 'w') clnsig_out_file = open("%s.clnsig" % base_fname, 'w') for c in range(len(chr_names)): chr = chr_names[c] chr_full_name = chr_full_names[c] assert chr in chr_dic chr_seq = chr_dic[chr] chr_len = len(chr_seq) if chr in genotype_genes: chr_genes = genotype_genes[chr] chr_genes = sorted(chr_genes, key=lambda x: (x[1], x[2], x[3])) else: chr_genes = [] chr_genotype_vars = [] chr_genotype_vari = 0 if graph_index: if chr in genotype_vars: chr_genotype_vars = genotype_vars[chr] chr_genotype_haplotypes = [] chr_genotype_hti = 0 if chr in genotype_haplotypes: chr_genotype_haplotypes = genotype_haplotypes[chr] def add_vars(left, right, chr_genotype_vari, chr_genotype_hti, haplotype_num): # Output variants with clinical significance while chr_genotype_vari < len(chr_genotype_vars): var_left, \ var_type, \ var_data, \ var_id \ = chr_genotype_vars[chr_genotype_vari] var_right = var_left if var_type == "deletion": var_right += var_data if var_right > right: break if var_right >= left: chr_genotype_vari += 1 continue out_str = "%s\t%s\t%s\t%d\t%s" % (var_id, var_type, chr, var_left + off, var_data) print(out_str, file=var_out_file) print(out_str, file=index_var_out_file) if var_id in genotype_clnsig: var_gene, clnsig = genotype_clnsig[var_id] print("%s\t%s\t%s" \ % (var_id, var_gene, clnsig), file=clnsig_out_file) chr_genotype_vari += 1 # Output haplotypes while chr_genotype_hti < len(chr_genotype_haplotypes): ht_left, ht_right, ht_vars = chr_genotype_haplotypes[ chr_genotype_hti] if ht_right > right: break if ht_right >= left: chr_genotype_hti += 1 continue print("ht%d\t%s\t%d\t%d\t%s" \ % (haplotype_num, chr, ht_left + off, ht_right + off, ','.join(ht_vars)), file=haplotype_out_file) chr_genotype_hti += 1 haplotype_num += 1 return chr_genotype_vari, chr_genotype_hti, haplotype_num out_chr_seq = "" off = 0 prev_right = 0 for gene in chr_genes: left, right, length, name, family, exon_str, strand = gene if not graph_index: # Output gene (genotype_genome.gene) print("%s\t%s\t%s\t%d\t%d\t%s\t%s" \ % (family.upper(), name, chr, left, right, exon_str, strand), file=locus_out_file) continue chr_genotype_vari, \ chr_genotype_hti, \ haplotype_num \ = add_vars(left, right, chr_genotype_vari, chr_genotype_hti, haplotype_num) # Read gene family sequences and information allele_seqs = typing_common.read_allele_seq("%s_backbone.fa" % family) allele_vars = typing_common.read_variants("%s.snp" % family) allele_index_vars = typing_common.read_variants("%s.index.snp" % family) allele_haplotypes = typing_common.read_haplotypes("%s.haplotype" % family) links = typing_common.read_links("%s.link" % family, True) if name not in allele_seqs: continue if name not in allele_vars or name not in allele_index_vars: vars = [] index_vars = [] else: vars = allele_vars[name] index_vars = allele_index_vars[name] allele_seq = allele_seqs[name] index_var_ids = set() for _, _, _, var_id in index_vars: index_var_ids.add(var_id) if name not in allele_haplotypes: haplotypes = [] else: haplotypes = allele_haplotypes[name] assert length == len(allele_seq) assert left < chr_len and right < chr_len # Skipping overlapping genes if left < prev_right: print("Warning: skipping %s ..." % (name), file=sys.stderr) continue varID2htID = {} assert left < right prev_length = right - left + 1 assert prev_length <= length if prev_right < left: out_chr_seq += chr_seq[prev_right:left] # Output gene (genotype_genome.locus) print("%s\t%s\t%s\t%d\t%d\t%s\t%s" \ % (family.upper(), name, chr, len(out_chr_seq), len(out_chr_seq) + length - 1, exon_str, strand), file=locus_out_file) # Output coord (genotype_genome.coord) print("%s\t%d\t%d\t%d" \ % (chr, len(out_chr_seq), left, right - left + 1), file=coord_out_file) out_chr_seq += allele_seq # Output variants (genotype_genome.snp and genotype_genome.index.snp) for var in vars: var_left, var_type, var_data, var_id = var new_var_id = "hv%d" % var_num varID2htID[var_id] = new_var_id new_var_left = var_left + left + off assert var_type in ["single", "deletion", "insertion"] assert new_var_left < len(out_chr_seq) if var_type == "single": assert out_chr_seq[new_var_left] != var_data elif var_type == "deletion": assert new_var_left + var_data <= len(out_chr_seq) else: assert var_type == "insertion" out_str = "%s\t%s\t%s\t%d\t%s" \ % (new_var_id, var_type, chr, new_var_left, var_data) print(out_str, file=var_out_file) if var_id in index_var_ids: print(out_str, file=index_var_out_file) var_num += 1 # Output haplotypes (genotype_genome.haplotype) for haplotype in haplotypes: ht_left, ht_right, ht_vars = haplotype new_ht_left = ht_left + left + off assert new_ht_left < len(out_chr_seq) new_ht_right = ht_right + left + off assert new_ht_left <= new_ht_right assert new_ht_right <= len(out_chr_seq) new_ht_vars = [] for var_id in ht_vars: assert var_id in varID2htID new_ht_vars.append(varID2htID[var_id]) print("ht%d\t%s\t%d\t%d\t%s" \ % (haplotype_num, chr, new_ht_left, new_ht_right, ','.join(new_ht_vars)), file=haplotype_out_file) haplotype_num += 1 # Output link information between alleles and variants (genotype_genome.link) for link in links: var_id, allele_names = link if var_id not in varID2htID: continue new_var_id = varID2htID[var_id] print("%s\t%s" % (new_var_id, " ".join(allele_names)), file=link_out_file) off += (length - prev_length) prev_right = right + 1 if not graph_index: continue # Write the rest of the Vars chr_genotype_vari, \ chr_genotype_hti, \ haplotype_num \ = add_vars(sys.maxsize, sys.maxsize, chr_genotype_vari, chr_genotype_hti, haplotype_num) print("%s\t%d\t%d\t%d" \ % (chr, len(out_chr_seq), prev_right, len(chr_seq) - prev_right), file=coord_out_file) out_chr_seq += chr_seq[prev_right:] assert len(out_chr_seq) == len(chr_seq) + off # Output chromosome sequence print(">%s" % (chr_full_name), file=genome_out_file) line_width = 60 for s in range(0, len(out_chr_seq), line_width): print(out_chr_seq[s:s + line_width], file=genome_out_file) genome_out_file.close() locus_out_file.close() var_out_file.close() index_var_out_file.close() haplotype_out_file.close() link_out_file.close() coord_out_file.close() clnsig_out_file.close() allele_out_file = open("%s.allele" % base_fname, 'w') if graph_index: for database in database_list: for line in open("%s.allele" % database): allele_name = line.strip() print("%s\t%s" % (database.upper(), allele_name), file=allele_out_file) allele_out_file.close() partial_out_file = open("%s.partial" % base_fname, 'w') if graph_index: for database in database_list: for line in open("%s.partial" % database): allele_name = line.strip() print("%s\t%s" % (database.upper(), allele_name), file=partial_out_file) partial_out_file.close() if not graph_index: shutil.copyfile("genome.fa", "%s.fa" % base_fname) # Index genotype_genome.fa index_cmd = ["samtools", "faidx", "%s.fa" % base_fname] subprocess.call(index_cmd) # Build indexes based on the above information if graph_index: assert aligner == "hisat2" build_cmd = [ "hisat2-build", "-p", str(threads), "--snp", "%s.index.snp" % base_fname, "--haplotype", "%s.haplotype" % base_fname, "%s.fa" % base_fname, "%s" % base_fname ] else: assert aligner in ["hisat2", "bowtie2"] build_cmd = [ "%s-build" % aligner, "-p" if aligner == "hisat2" else "--threads", str(threads), "%s.fa" % base_fname, "%s" % base_fname ] if verbose: print("\tRunning:", ' '.join(build_cmd), file=sys.stderr) subprocess.call(build_cmd, stdout=open("/dev/null", 'w'), stderr=open("/dev/null", 'w')) if aligner == "hisat2": index_fnames = ["%s.%d.ht2" % (base_fname, i + 1) for i in range(8)] else: index_fnames = ["%s.%d.bt2" % (base_fname, i + 1) for i in range(4)] index_fnames += [ "%s.rev.%d.bt2" % (base_fname, i + 1) for i in range(2) ] if not typing_common.check_files(index_fnames): print("Error: indexing failed! "\ "Perhaps, you may have forgotten to build %s executables?" \ % aligner, file=sys.stderr) sys.exit(1)
def genotype(base_fname, fastq, read_fnames, threads, num_mismatch, verbose, daehwan_debug): # Load genomic sequences chr_dic, chr_names, chr_full_names = typing_common.read_genome(open("%s.fa" % base_fname)) # variants, backbone sequence, and other sequeces genotype_fnames = ["%s.fa" % base_fname, "%s.gene" % base_fname, "%s.snp" % base_fname, "%s.index.snp" % base_fname, "%s.haplotype" % base_fname, "%s.link" % base_fname, "%s.coord" % base_fname, "%s.clnsig" % base_fname] # hisat2 graph index files genotype_fnames += ["%s.%d.ht2" % (base_fname, i+1) for i in range(8)] if not typing_common.check_files(genotype_fnames): print >> sys.stderr, "Error: some of the following files are missing!" for fname in genotype_fnames: print >> sys.stderr, "\t%s" % fname sys.exit(1) # Align reads, and sort the alignments into a BAM file align_reads(base_fname, read_fnames, fastq, threads, verbose) # Read HLA alleles (names and sequences) genes, gene_loci, gene_seqs = {}, {}, {} for line in open("%s.gene" % base_fname): family, allele_name, chr, left, right = line.strip().split() gene_name = "%s-%s" % (family, allele_name.split('*')[0]) assert gene_name not in genes genes[gene_name] = allele_name left, right = int(left), int(right) """ exons = [] for exon in exon_str.split(','): exon_left, exon_right = exon.split('-') exons.append([int(exon_left), int(exon_right)]) """ gene_loci[gene_name] = [allele_name, chr, left, right] assert chr in chr_dic chr_seq = chr_dic[chr] assert left < right assert right < len(chr_seq) gene_seqs[gene_name] = chr_dic[chr][left:right+1] # Read link information Links, var_genes, allele_vars = {}, {}, {} for line in open("%s.link" % base_fname): var_id, alleles = line.strip().split('\t') alleles = alleles.split() assert not var_id in Links Links[var_id] = alleles for allele in alleles: if allele not in allele_vars: allele_vars[allele] = set() allele_vars[allele].add(var_id) gene_name = "HLA-%s" % (allele.split('*')[0]) var_genes[var_id] = gene_name # gene alleles allele_names = {} for gene_name in genes.keys(): if gene_name not in allele_names: allele_names[gene_name] = [] gene_name2 = gene_name.split('-')[1] for allele_name in allele_vars.keys(): allele_name1 = allele_name.split('*')[0] if gene_name2 == allele_name1: allele_names[gene_name].append(allele_name) # Read HLA variants, and link information Vars, Var_list = {}, {} for line in open("%s.snp" % base_fname): var_id, var_type, chr, pos, data = line.strip().split('\t') pos = int(pos) # daehwan - for debugging purposes if var_id not in var_genes: continue assert var_id in var_genes gene_name = var_genes[var_id] if not gene_name in Vars: Vars[gene_name] = {} assert not gene_name in Var_list Var_list[gene_name] = [] assert not var_id in Vars[gene_name] Vars[gene_name][var_id] = [var_type, pos, data] Var_list[gene_name].append([pos, var_id]) for gene_name, in_var_list in Var_list.items(): Var_list[gene_name] = sorted(in_var_list) def lower_bound(Var_list, pos): low, high = 0, len(Var_list) while low < high: m = (low + high) / 2 m_pos = Var_list[m][0] if m_pos < pos: low = m + 1 elif m_pos > pos: high = m else: assert m_pos == pos while m > 0: if Var_list[m-1][0] < pos: break m -= 1 return m return low # HLA gene allele lengths """ HLA_lengths = {} for HLA_gene, HLA_alleles in HLAs.items(): HLA_lengths[HLA_gene] = {} for allele_name, seq in HLA_alleles.items(): HLA_lengths[HLA_gene][allele_name] = len(seq) """ # Cigar regular expression cigar_re = re.compile('\d+\w') test_list = [[sorted(genes.keys())]] for test_i in range(len(test_list)): test_HLA_list = test_list[test_i] for test_HLA_names in test_HLA_list: print >> sys.stderr, "\t%s" % (test_HLA_names) for gene in test_HLA_names: ref_allele = genes[gene] ref_seq = gene_seqs[gene] # ref_exons = refHLA_loci[gene][-1] # Read alignments alignview_cmd = ["samtools", "view"] alignview_cmd += ["hla_output.bam"] base_locus = 0 _, chr, left, right = gene_loci[gene] base_locus = left alignview_cmd += ["%s:%d-%d" % (chr, left + 1, right + 1)] bamview_proc = subprocess.Popen(alignview_cmd, stdout=subprocess.PIPE, stderr=open("/dev/null", 'w')) sort_read_cmd = ["sort", "-k", "1", "-n"] alignview_proc = subprocess.Popen(sort_read_cmd, stdin=bamview_proc.stdout, stdout=subprocess.PIPE, stderr=open("/dev/null", 'w')) # Count alleles HLA_counts, HLA_cmpt = {}, {} coverage = [0 for i in range(len(ref_seq) + 1)] num_reads, total_read_len = 0, 0 prev_read_id = None prev_exon = False for line in alignview_proc.stdout: cols = line.strip().split() read_id, flag, chr, pos, mapQ, cigar_str = cols[:6] origin_read_id = read_id if read_id.find('|') != -1: tmp_read_id = read_id.split('|')[0] try: read_id = int(tmp_read_id) except ValueError: None read_seq, qual = cols[9], cols[10] num_reads += 1 total_read_len += len(read_seq) flag, pos = int(flag), int(pos) pos -= 1 if pos < 0: continue if flag & 0x4 != 0: continue NM, Zs, MD = "", "", "" for i in range(11, len(cols)): col = cols[i] if col.startswith("Zs"): Zs = col[5:] elif col.startswith("MD"): MD = col[5:] elif col.startswith("NM"): NM = int(col[5:]) if NM > num_mismatch: continue # daehwan - for debugging purposes debug = False if read_id in ["2339"] and False: debug = True print "read_id: %s)" % read_id, pos, cigar_str, "NM:", NM, MD, Zs print " ", read_seq vars = [] if Zs: vars = Zs.split(',') assert MD != "" MD_str_pos, MD_len = 0, 0 read_pos, left_pos = 0, pos right_pos = left_pos cigars = cigar_re.findall(cigar_str) cigars = [[cigar[-1], int(cigar[:-1])] for cigar in cigars] cmp_list = [] for i in range(len(cigars)): cigar_op, length = cigars[i] if cigar_op == 'M': first = True MD_len_used = 0 while True: if not first or MD_len == 0: if MD[MD_str_pos].isdigit(): num = int(MD[MD_str_pos]) MD_str_pos += 1 while MD_str_pos < len(MD): if MD[MD_str_pos].isdigit(): num = num * 10 + int(MD[MD_str_pos]) MD_str_pos += 1 else: break MD_len += num # Insertion or full match followed if MD_len >= length: MD_len -= length cmp_list.append(["match", right_pos + MD_len_used, length - MD_len_used]) break first = False read_base = read_seq[read_pos + MD_len] MD_ref_base = MD[MD_str_pos] MD_str_pos += 1 assert MD_ref_base in "ACGT" cmp_list.append(["match", right_pos + MD_len_used, MD_len - MD_len_used]) cmp_list.append(["mismatch", right_pos + MD_len, 1]) MD_len_used = MD_len + 1 MD_len += 1 # Full match if MD_len == length: MD_len = 0 break elif cigar_op == 'I': cmp_list.append(["insertion", right_pos, length]) elif cigar_op == 'D': if MD[MD_str_pos] == '0': MD_str_pos += 1 assert MD[MD_str_pos] == '^' MD_str_pos += 1 while MD_str_pos < len(MD): if not MD[MD_str_pos] in "ACGT": break MD_str_pos += 1 cmp_list.append(["deletion", right_pos, length]) elif cigar_op == 'S': cmp_list.append(["soft", right_pos, length]) else: assert cigar_op == 'N' cmp_list.append(["intron", right_pos, length]) if cigar_op in "MND": right_pos += length if cigar_op in "MIS": read_pos += length """ exon = False for exon in ref_exons: exon_left, exon_right = exon if right_pos <= exon_left or pos > exon_right: continue else: exon = True break """ if left_pos < base_locus or \ right_pos - base_locus > len(ref_seq): continue def add_stat(HLA_cmpt, HLA_counts, HLA_count_per_read, exon = True): max_count = max(HLA_count_per_read.values()) cur_cmpt = set() for allele, count in HLA_count_per_read.items(): if count < max_count: continue """ if allele in exclude_allele_list: continue """ cur_cmpt.add(allele) if not allele in HLA_counts: HLA_counts[allele] = 1 else: HLA_counts[allele] += 1 if len(cur_cmpt) == 0: return # daehwan - for debugging purposes alleles = ["", ""] # alleles = ["B*40:304", "B*40:02:01"] allele1_found, allele2_found = False, False for allele, count in HLA_count_per_read.items(): if count < max_count: continue if allele == alleles[0]: allele1_found = True elif allele == alleles[1]: allele2_found = True if allele1_found != allele2_found: print alleles[0], HLA_count_per_read[alleles[0]] print alleles[1], HLA_count_per_read[alleles[1]] if allele1_found: print ("%s\tread_id %s - %d vs. %d]" % (alleles[0], prev_read_id, max_count, HLA_count_per_read[alleles[1]])) else: print ("%s\tread_id %s - %d vs. %d]" % (alleles[1], prev_read_id, max_count, HLA_count_per_read[alleles[0]])) print read_seq cur_cmpt = sorted(list(cur_cmpt)) cur_cmpt = '-'.join(cur_cmpt) add = 1 """ if partial and not exon: add *= 0.2 """ if not cur_cmpt in HLA_cmpt: HLA_cmpt[cur_cmpt] = add else: HLA_cmpt[cur_cmpt] += add if read_id != prev_read_id: if prev_read_id != None: add_stat(HLA_cmpt, HLA_counts, HLA_count_per_read, prev_exon) HLA_count_per_read = {} for HLA_name in allele_names[gene]: if HLA_name.find("BACKBONE") != -1: continue HLA_count_per_read[HLA_name] = 0 def add_count(var_id, add): assert var_id in Links alleles = Links[var_id] for allele in alleles: if allele.find("BACKBONE") != -1: continue HLA_count_per_read[allele] += add # daehwan - for debugging purposes if debug: if allele in ["DQA1*05:05:01:01", "DQA1*05:05:01:02"]: print allele, add, var_id # Decide which allele(s) a read most likely came from # also sanity check - read length, cigar string, and MD string for var_id, data in Vars[gene].items(): var_type, var_pos, var_data = data if var_type != "deletion": continue if left_pos >= var_pos and right_pos <= var_pos + int(var_data): add_count(var_id, -1) ref_pos, read_pos, cmp_cigar_str, cmp_MD = left_pos, 0, "", "" cigar_match_len, MD_match_len = 0, 0 for cmp in cmp_list: type = cmp[0] length = cmp[2] if type == "match": var_idx = lower_bound(Var_list[gene], ref_pos) while var_idx < len(Var_list[gene]): var_pos, var_id = Var_list[gene][var_idx] if ref_pos + length <= var_pos: break if ref_pos <= var_pos: var_type, _, var_data = Vars[gene][var_id] if var_type == "insertion": if ref_pos < var_pos and ref_pos + length > var_pos + len(var_data): add_count(var_id, -1) # daehwan - for debugging purposes if debug: print cmp, var_id, Links[var_id] elif var_type == "deletion": del_len = int(var_data) if ref_pos < var_pos and ref_pos + length > var_pos + del_len: # daehwan - for debugging purposes if debug: print cmp, var_id, Links[var_id], -1, Vars[gene][var_id] # Check if this might be one of the two tandem repeats (the same left coordinate) cmp_left, cmp_right = cmp[1], cmp[1] + cmp[2] test1_seq1 = ref_seq[cmp_left-base_locus:cmp_right-base_locus] test1_seq2 = ref_seq[cmp_left-base_locus:var_pos-base_locus] + ref_seq[var_pos + del_len - base_locus:cmp_right + del_len - base_locus] # Check if this happens due to small repeats (the same right coordinate - e.g. 19 times of TTTC in DQA1*05:05:01:02) cmp_left -= read_pos cmp_right += (len(read_seq) - read_pos - cmp[2]) test2_seq1 = ref_seq[cmp_left+int(var_data)-base_locus:cmp_right-base_locus] test2_seq2 = ref_seq[cmp_left-base_locus:var_pos-base_locus] + ref_seq[var_pos+int(var_data)-base_locus:cmp_right-base_locus] if test1_seq1 != test1_seq2 and test2_seq1 != test2_seq2: add_count(var_id, -1) else: if debug: print cmp, var_id, Links[var_id], -1 add_count(var_id, -1) var_idx += 1 read_pos += length ref_pos += length cigar_match_len += length MD_match_len += length elif type == "mismatch": read_base = read_seq[read_pos] var_idx = lower_bound(Var_list[gene], ref_pos) while var_idx < len(Var_list[gene]): var_pos, var_id = Var_list[gene][var_idx] if ref_pos < var_pos: break if ref_pos == var_pos: var_type, _, var_data = Vars[gene][var_id] if var_type == "single": if var_data == read_base: # daehwan - for debugging purposes if debug: print cmp, var_id, 1, var_data, read_base, Links[var_id] # daehwan - for debugging purposes if False: read_qual = ord(qual[read_pos]) add_count(var_id, (read_qual - 60) / 60.0) else: add_count(var_id, 1) # daehwan - check out if this routine is appropriate # else: # add_count(var_id, -1) var_idx += 1 cmp_MD += ("%d%s" % (MD_match_len, ref_seq[ref_pos-base_locus])) MD_match_len = 0 cigar_match_len += 1 read_pos += 1 ref_pos += 1 elif type == "insertion": ins_seq = read_seq[read_pos:read_pos+length] var_idx = lower_bound(Var_list[gene], ref_pos) # daehwan - for debugging purposes if debug: print left_pos, cigar_str, MD, vars print ref_pos, ins_seq, Var_list[gene][var_idx], Vars[gene][Var_list[gene][var_idx][1]] # sys.exit(1) while var_idx < len(Var_list[gene]): var_pos, var_id = Var_list[gene][var_idx] if ref_pos < var_pos: break if ref_pos == var_pos: var_type, _, var_data = Vars[gene][var_id] if var_type == "insertion": if var_data == ins_seq: # daehwan - for debugging purposes if debug: print cmp, var_id, 1, Links[var_id] add_count(var_id, 1) var_idx += 1 if cigar_match_len > 0: cmp_cigar_str += ("%dM" % cigar_match_len) cigar_match_len = 0 read_pos += length cmp_cigar_str += ("%dI" % length) elif type == "deletion": del_len = length # Deletions can be shifted bidirectionally temp_ref_pos = ref_pos while temp_ref_pos > 0: last_bp = ref_seq[temp_ref_pos + del_len - 1 - base_locus] prev_bp = ref_seq[temp_ref_pos - 1 - base_locus] if last_bp != prev_bp: break temp_ref_pos -= 1 var_idx = lower_bound(Var_list[gene], temp_ref_pos) while var_idx < len(Var_list[gene]): var_pos, var_id = Var_list[gene][var_idx] if temp_ref_pos < var_pos: first_bp = ref_seq[temp_ref_pos - base_locus] next_bp = ref_seq[temp_ref_pos + del_len - base_locus] if first_bp == next_bp: temp_ref_pos += 1 continue else: break if temp_ref_pos == var_pos: var_type, _, var_data = Vars[gene][var_id] if var_type == "deletion": var_len = int(var_data) if var_len == length: if debug: print cmp, var_id, 1, Links[var_id] print ref_seq[var_pos - 10-base_locus:var_pos-base_locus], ref_seq[var_pos-base_locus:var_pos+int(var_data)-base_locus], ref_seq[var_pos+int(var_data)-base_locus:var_pos+int(var_data)+10-base_locus] add_count(var_id, 1) var_idx += 1 if cigar_match_len > 0: cmp_cigar_str += ("%dM" % cigar_match_len) cigar_match_len = 0 cmp_MD += ("%d" % MD_match_len) MD_match_len = 0 cmp_cigar_str += ("%dD" % length) cmp_MD += ("^%s" % ref_seq[ref_pos-base_locus:ref_pos+length-base_locus]) ref_pos += length elif type == "soft": if cigar_match_len > 0: cmp_cigar_str += ("%dM" % cigar_match_len) cigar_match_len = 0 read_pos += length cmp_cigar_str += ("%dS" % length) else: assert type == "intron" if cigar_match_len > 0: cmp_cigar_str += ("%dM" % cigar_match_len) cigar_match_len = 0 cmp_cigar_str += ("%dN" % length) ref_pos += length if cigar_match_len > 0: cmp_cigar_str += ("%dM" % cigar_match_len) cmp_MD += ("%d" % MD_match_len) if read_pos != len(read_seq) or \ cmp_cigar_str != cigar_str or \ cmp_MD != MD: print >> sys.stderr, "Error:", cigar_str, MD print >> sys.stderr, "\tcomputed:", cmp_cigar_str, cmp_MD print >> sys.stderr, "\tcmp list:", cmp_list assert False prev_read_id = read_id # prev_exon = exon if num_reads <= 0: continue if prev_read_id != None: add_stat(HLA_cmpt, HLA_counts, HLA_count_per_read) HLA_counts = [[allele, count] for allele, count in HLA_counts.items()] def HLA_count_cmp(a, b): if a[1] != b[1]: return b[1] - a[1] assert a[0] != b[0] if a[0] < b[0]: return -1 else: return 1 HLA_counts = sorted(HLA_counts, cmp=HLA_count_cmp) for count_i in range(len(HLA_counts)): count = HLA_counts[count_i] print >> sys.stderr, "\t\t\t\t%d %s (count: %d)" % (count_i + 1, count[0], count[1]) if count_i >= 9: break print >> sys.stderr def normalize(prob): total = sum(prob.values()) for allele, mass in prob.items(): prob[allele] = mass / total def normalize2(prob, length): total = 0 for allele, mass in prob.items(): assert allele in length total += (mass / length[allele]) for allele, mass in prob.items(): assert allele in length prob[allele] = mass / length[allele] / total def prob_diff(prob1, prob2): diff = 0.0 for allele in prob1.keys(): if allele in prob2: diff += abs(prob1[allele] - prob2[allele]) else: diff += prob1[allele] return diff def HLA_prob_cmp(a, b): if a[1] != b[1]: if a[1] < b[1]: return 1 else: return -1 assert a[0] != b[0] if a[0] < b[0]: return -1 else: return 1 HLA_prob, HLA_prob_next = {}, {} for cmpt, count in HLA_cmpt.items(): alleles = cmpt.split('-') for allele in alleles: if allele not in HLA_prob: HLA_prob[allele] = 0.0 HLA_prob[allele] += (float(count) / len(alleles)) """ assert gene in HLA_lengths HLA_length = HLA_lengths[gene] """ HLA_length = {} # normalize2(HLA_prob, HLA_length) normalize(HLA_prob) def next_prob(HLA_cmpt, HLA_prob, HLA_length): HLA_prob_next = {} for cmpt, count in HLA_cmpt.items(): alleles = cmpt.split('-') alleles_prob = 0.0 for allele in alleles: assert allele in HLA_prob alleles_prob += HLA_prob[allele] for allele in alleles: if allele not in HLA_prob_next: HLA_prob_next[allele] = 0.0 HLA_prob_next[allele] += (float(count) * HLA_prob[allele] / alleles_prob) # normalize2(HLA_prob_next, HLA_length) normalize(HLA_prob_next) return HLA_prob_next diff, iter = 1.0, 0 while diff > 0.0001 and iter < 1000: HLA_prob_next = next_prob(HLA_cmpt, HLA_prob, HLA_length) diff = prob_diff(HLA_prob, HLA_prob_next) HLA_prob = HLA_prob_next iter += 1 """ for allele, prob in HLA_prob.items(): allele_len = len(HLAs[gene][allele]) HLA_prob[allele] /= float(allele_len) normalize(HLA_prob) """ HLA_prob = [[allele, prob] for allele, prob in HLA_prob.items()] HLA_prob = sorted(HLA_prob, cmp=HLA_prob_cmp) success = [False for i in range(len(test_HLA_names))] found_list = [False for i in range(len(test_HLA_names))] for prob_i in range(len(HLA_prob)): prob = HLA_prob[prob_i] print >> sys.stderr, "\t\t\t\t%d ranked %s (abundance: %.2f%%)" % (prob_i + 1, prob[0], prob[1] * 100.0) if prob_i >= 9: break print >> sys.stderr """ if len(test_HLA_names) == 2: HLA_prob, HLA_prob_next = {}, {} for cmpt, count in HLA_cmpt.items(): alleles = cmpt.split('-') for allele1 in alleles: for allele2 in HLA_names[gene]: if allele1 < allele2: allele_pair = "%s-%s" % (allele1, allele2) else: allele_pair = "%s-%s" % (allele2, allele1) if not allele_pair in HLA_prob: HLA_prob[allele_pair] = 0.0 HLA_prob[allele_pair] += (float(count) / len(alleles)) if len(HLA_prob) <= 0: continue # Choose top allele pairs def choose_top_alleles(HLA_prob): HLA_prob_list = [[allele_pair, prob] for allele_pair, prob in HLA_prob.items()] HLA_prob_list = sorted(HLA_prob_list, cmp=HLA_prob_cmp) HLA_prob = {} best_prob = HLA_prob_list[0][1] for i in range(len(HLA_prob_list)): allele_pair, prob = HLA_prob_list[i] if prob * 2 <= best_prob: break HLA_prob[allele_pair] = prob normalize(HLA_prob) return HLA_prob HLA_prob = choose_top_alleles(HLA_prob) def next_prob(HLA_cmpt, HLA_prob): HLA_prob_next = {} for cmpt, count in HLA_cmpt.items(): alleles = cmpt.split('-') prob = 0.0 for allele in alleles: for allele_pair in HLA_prob.keys(): if allele in allele_pair: prob += HLA_prob[allele_pair] for allele in alleles: for allele_pair in HLA_prob.keys(): if not allele in allele_pair: continue if allele_pair not in HLA_prob_next: HLA_prob_next[allele_pair] = 0.0 HLA_prob_next[allele_pair] += (float(count) * HLA_prob[allele_pair] / prob) normalize(HLA_prob_next) return HLA_prob_next diff, iter = 1.0, 0 while diff > 0.0001 and iter < 1000: HLA_prob_next = next_prob(HLA_cmpt, HLA_prob) diff = prob_diff(HLA_prob, HLA_prob_next) HLA_prob = HLA_prob_next HLA_prob = choose_top_alleles(HLA_prob) iter += 1 HLA_prob = [[allele_pair, prob] for allele_pair, prob in HLA_prob.items()] HLA_prob = sorted(HLA_prob, cmp=HLA_prob_cmp) success = [False] for prob_i in range(len(HLA_prob)): allele_pair, prob = HLA_prob[prob_i] allele1, allele2 = allele_pair.split('-') if best_alleles and prob_i < 1: print >> sys.stdout, "PairModel %s (abundance: %.2f%%)" % (allele_pair, prob * 100.0) if simulation: if allele1 in test_HLA_names and allele2 in test_HLA_names: rank_i = prob_i while rank_i > 0: if HLA_prob[rank_i-1][1] == prob: rank_i -= 1 else: break print >> sys.stderr, "\t\t\t*** %d ranked %s (abundance: %.2f%%)" % (rank_i + 1, allele_pair, prob * 100.0) if rank_i == 0: success[0] = True break print >> sys.stderr, "\t\t\t\t%d ranked %s (abundance: %.2f%%)" % (prob_i + 1, allele_pair, prob * 100.0) if not simulation and prob_i >= 9: break print >> sys.stderr """ # Read variants with clinical significance clnsigs = {} for line in open("%s.clnsig" % base_fname): var_id, var_gene, var_clnsig = line.strip().split('\t') clnsigs[var_id] = [var_gene, var_clnsig] vars, Var_list = {}, {} for line in open("%s.snp" % base_fname): var_id, type, chr, left, data = line.strip().split() if var_id not in clnsigs: continue left = int(left) if type == "deletion": data = int(data) vars[var_id] = [chr, left, type, data] if chr not in Var_list: Var_list[chr] = [] Var_list[chr].append([left, var_id]) var_counts = {} # Read alignments alignview_cmd = ["samtools", "view", "hla_output.bam"] bamview_proc = subprocess.Popen(alignview_cmd, stdout=subprocess.PIPE, stderr=open("/dev/null", 'w')) for line in bamview_proc.stdout: cols = line.strip().split() read_id, flag, chr, pos, mapQ, cigar_str = cols[:6] read_seq, qual = cols[9], cols[10] flag, pos = int(flag), int(pos) pos -= 1 if pos < 0: continue if flag & 0x4 != 0: continue if chr not in Var_list: continue assert chr in chr_dic chr_seq = chr_dic[chr] NM, Zs, MD, NH = "", "", "", "" for i in range(11, len(cols)): col = cols[i] if col.startswith("Zs"): Zs = col[5:] elif col.startswith("MD"): MD = col[5:] elif col.startswith("NM"): NM = int(col[5:]) elif col.startswith("NH"): NH = int(col[5:]) assert NH != "" NH = int(NH) if NH > 1: continue if NM > num_mismatch: continue read_vars = [] if Zs: read_vars = Zs.split(',') for read_var in read_vars: _, _, var_id = read_var.split('|') if var_id not in clnsigs: continue if var_id not in var_counts: var_counts[var_id] = [1, 0] else: var_counts[var_id][0] += 1 assert MD != "" MD_str_pos, MD_len = 0, 0 read_pos, left_pos = 0, pos right_pos = left_pos cigars = cigar_re.findall(cigar_str) cigars = [[cigar[-1], int(cigar[:-1])] for cigar in cigars] cmp_list = [] for i in range(len(cigars)): cigar_op, length = cigars[i] if cigar_op == 'M': chr_var_list = Var_list[chr] var_idx = lower_bound(chr_var_list, right_pos) while var_idx < len(chr_var_list): var_pos, var_id = chr_var_list[var_idx] if var_pos >= right_pos + length: break if var_pos >= right_pos: assert var_id in vars _, _, var_type, var_data = vars[var_id] contradict = False if var_type == "single": contradict = (read_seq[read_pos + var_pos - right_pos] == chr_seq[var_pos]) elif var_type == "insertion": contradict = (right_pos < var_pos) else: contradict = True if contradict: if var_id not in var_counts: var_counts[var_id] = [0, 1] else: var_counts[var_id][1] += 1 var_idx += 1 if cigar_op in "MND": right_pos += length if cigar_op in "MIS": read_pos += length for var_id, counts in var_counts.items(): if counts[0] < 2: # or counts[0] * 3 < counts[1]: continue assert var_id in vars var_chr, var_left, var_type, var_data = vars[var_id] assert var_id in clnsigs var_gene, var_clnsig = clnsigs[var_id] print >> sys.stderr, "\t\t\t%s %s: %s:%d %s %s (%s): %d-%d" % \ (var_gene, var_id, var_chr, var_left, var_type, var_data, var_clnsig, counts[0], counts[1])
def extract_reads(base_fname, # base file name for index, variants, haplotypes, etc. /Users/katiecampbell/hisat_test/tm/hisat2-hisat2_v2.2.0_beta/evaluation/hla-analysis/genotype_genome database_list, # comma-separated, use hla read_dir, out_dir, suffix, read_fname, fastq, paired, simulation, threads, threads_aprocess, max_sample, job_range, aligner, block_size, is_rna, rna_strandness, verbose): if block_size > 0: resource.setrlimit(resource.RLIMIT_NOFILE, (1000, 1000)) resource.setrlimit(resource.RLIMIT_NPROC, (1000, 1000)) genotype_fnames = ["%s.fa" % base_fname, # Filenames for genotypes in /Users/katiecampbell/hisat_test/tm/hisat2-hisat2_v2.2.0_beta/evaluation/hla-analysis/genotype_genome "%s.locus" % base_fname, "%s.snp" % base_fname, "%s.haplotype" % base_fname, "%s.link" % base_fname, "%s.coord" % base_fname, "%s.clnsig" % base_fname] # graph index files if aligner == "hisat2": genotype_fnames += ["%s.%d.ht2" % (base_fname, i+1) for i in range(8)] # gets the hisat index files else: assert aligner == "bowtie2" genotype_fnames = ["%s.%d.bt2" % (base_fname, i+1) for i in range(4)] genotype_fnames += ["%s.rev.%d.bt2" % (base_fname, i+1) for i in range(2)] if not typing_common.check_files(genotype_fnames): # Checks to see if file exists print >> sys.stderr, "Error: %s related files do not exist as follows:" % base_fname for fname in genotype_fnames: print >> sys.stderr, "\t%s" % fname sys.exit(1) filter_region = len(database_list) > 0 ranges = [] regions, region_loci = {}, {} for line in open("%s.locus" % base_fname): family, allele_name, chr, left, right = line.strip().split()[:5] # pulls HLA families in the if filter_region and family.lower() not in database_list: continue region_name = "%s-%s" % (family, allele_name.split('*')[0]) # specifies which HLA gene assert region_name not in regions regions[region_name] = allele_name left, right = int(left), int(right) """ exons = [] for exon in exon_str.split(','): exon_left, exon_right = exon.split('-') exons.append([int(exon_left), int(exon_right)]) """ if chr not in region_loci: region_loci[chr] = {} region_loci[chr][region_name] = [allele_name, chr, left, right] # Returns chromosomal region of chosen databases database_list.add(family.lower()) if out_dir != "" and not os.path.exists(out_dir): os.mkdir(out_dir) # Extract reads if len(read_fname) > 0: if paired: fq_fnames = [read_fname[0]] fq_fnames2 = [read_fname[1]] else: fq_fnames = read_fname else: if paired: fq_fnames = glob.glob("%s/*.1.%s" % (read_dir, suffix)) else: fq_fnames = glob.glob("%s/*.%s" % (read_dir, suffix)) count = 0 pids = [0 for i in range(threads)] for file_i in range(len(fq_fnames)): if file_i >= max_sample: break fq_fname = fq_fnames[file_i] if job_range[1] > 1: if job_range[0] != (file_i % job_range[1]): continue fq_fname_base = fq_fname.split('/')[-1] one_suffix = ".1." + suffix if fq_fname_base.find(one_suffix) != -1: fq_fname_base = fq_fname_base[:fq_fname_base.find(one_suffix)] else: fq_fname_base = fq_fname_base.split('.')[0] if paired: if read_dir == "": fq_fname2 = fq_fnames2[file_i] else: fq_fname2 = "%s/%s.2.%s" % (read_dir, fq_fname_base, suffix) if not os.path.exists(fq_fname2): print >> sys.stderr, "%s does not exist." % fq_fname2 continue else: fq_fname2 = "" if paired: if out_dir != "": if os.path.exists("%s/%s.extracted.1.fq.gz" % (out_dir, fq_fname_base)): continue else: if out_dir != "": if os.path.exists("%s/%s.extracted.fq.gz" % (out_dir, fq_fname_base)): continue count += 1 print >> sys.stderr, "\t%d: Extracting reads from %s" % (count, fq_fname_base) def work(fq_fname_base, fq_fname, fq_fname2, ranges, simulation, verbose): aligner_cmd = [aligner] if threads_aprocess > 1: aligner_cmd += ["-p", "%d" % threads_aprocess] if not fastq: aligner_cmd += ["-f"] aligner_cmd += ["-x", base_fname] if aligner == "hisat2": if not is_rna: aligner_cmd += ["--no-spliced-alignment"] else: if rna_strandness != "": aligner_cmd += ["--rna-strandness", rna_strandness] else: pass # aligner_cmd += ["--max-altstried", "64"] if not is_rna: aligner_cmd += ["-X", "1000"] if paired: aligner_cmd += ["-1", fq_fname, "-2", fq_fname2] else: aligner_cmd += ["-U", fq_fname] if verbose: print >> sys.stderr, "\t\trunning", ' '.join(aligner_cmd) align_proc = subprocess.Popen(aligner_cmd, stdout=subprocess.PIPE, stderr=open("/dev/null", 'w')) gzip_dic = {} out_dir_slash = out_dir if out_dir != "": out_dir_slash += "/" for database in database_list: if paired: # LP6005041-DNA_A01.extracted.1.fq.gz gzip1_proc = subprocess.Popen(["gzip"], stdin=subprocess.PIPE, stdout=open("%s%s.%s.extracted.1.fq.gz" % (out_dir_slash, fq_fname_base, database), 'w'), stderr=open("/dev/null", 'w')) # LP6005041-DNA_A01.extracted.2.fq.gz gzip2_proc = subprocess.Popen(["gzip"], stdin=subprocess.PIPE, stdout=open("%s%s.%s.extracted.2.fq.gz" % (out_dir_slash, fq_fname_base, database), 'w'), stderr=open("/dev/null", 'w')) else: # LP6005041-DNA_A01.extracted.fq.gz gzip1_proc = subprocess.Popen(["gzip"], stdin=subprocess.PIPE, stdout=open("%s%s.%s.extracted.fq.gz" % (out_dir_slash, fq_fname_base, database), 'w'), stderr=open("/dev/null", 'w')) gzip_dic[database] = [gzip1_proc, gzip2_proc if paired else None] whole_gzip_dic = {} if block_size > 0: mult = block_size / 1000000 for chr_line in open("%s.fa.fai" % base_fname): chr, length = chr_line.strip().split('\t')[:2] length = int(length) if chr not in [str(i+1) for i in range(22)] + ['X', 'Y', 'MT']: continue length = (length + block_size - 1) / block_size assert chr not in whole_gzip_dic whole_gzip_dic[chr] = [] for region_i in range(length): if paired: # LP6005041-DNA_A01.extracted.1.fq.gz gzip1_proc = subprocess.Popen(["gzip"], stdin=subprocess.PIPE, stdout=open("%s%s.%s.%d_%dM.extracted.1.fq.gz" % (out_dir_slash, fq_fname_base, chr, region_i * mult, (region_i + 1) * mult), 'w'), stderr=open("/dev/null", 'w')) # LP6005041-DNA_A01.extracted.2.fq.gz gzip2_proc = subprocess.Popen(["gzip"], stdin=subprocess.PIPE, stdout=open("%s%s.%s.%d_%dM.extracted.2.fq.gz" % (out_dir_slash, fq_fname_base, chr, region_i * mult, (region_i + 1) * mult), 'w'), stderr=open("/dev/null", 'w')) else: # LP6005041-DNA_A01.extracted.fq.gz gzip1_proc = subprocess.Popen(["gzip"], stdin=subprocess.PIPE, stdout=open("%s%s.%s.%d_%dM.extracted.fq.gz" % (out_dir_slash, fq_fname_base, chr, region_i * mult, (region_i + 1) * mult), 'w'), stderr=open("/dev/null", 'w')) whole_gzip_dic[chr].append([gzip1_proc, gzip2_proc if paired else None]) def write_read(gzip_proc, read_name, seq, qual): if fastq: gzip_proc.stdin.write("@%s\n" % read_name) gzip_proc.stdin.write("%s\n" % seq) gzip_proc.stdin.write("+\n") gzip_proc.stdin.write("%s\n" % qual) else: gzip_proc.stdin.write(">%s\n" % prev_read_name) gzip_proc.stdin.write("%s\n" % seq) prev_read_name, extract_read, whole_extract_read, read1, read2, read1_first, read2_first = "", set(), set(), [], [], True, True for line in align_proc.stdout: if line.startswith('@'): continue line = line.strip() cols = line.split() read_name, flag, chr, pos, mapQ, cigar, _, _, _, read, qual = cols[:11] flag, pos = int(flag), int(pos) - 1 strand = '-' if flag & 0x10 else '+' AS, XS, NH = "", "", "" for i in range(11, len(cols)): col = cols[i] if col.startswith("AS"): AS = int(col[5:]) elif col.startswith("XS"): XS = col[5:] elif col.startswith("NH"): NH = int(col[5:]) if (not simulation and read_name != prev_read_name) or \ (simulation and read_name.split('|')[0] != prev_read_name.split('|')[0]): for region in extract_read: print region print prev_read_name write_read(gzip_dic[region][0], prev_read_name, read1[0], read1[1]) if paired: write_read(gzip_dic[region][1], prev_read_name, read2[0], read2[1]) for chr_region_num in whole_extract_read: region_chr, region_num = chr_region_num.split('-') region_num = int(region_num) if region_chr not in whole_gzip_dic: continue assert region_num < len(whole_gzip_dic[region_chr]) write_read(whole_gzip_dic[region_chr][region_num][0], prev_read_name, read1[0], read1[1]) if paired: write_read(whole_gzip_dic[region_chr][region_num][1], prev_read_name, read2[0], read2[1]) prev_read_name, extract_read, whole_extract_read, read1, read2, read1_first, read2_first = read_name, set(), set(), [], [], True, True if flag & 0x4 == 0 and \ ((aligner == "hisat2" and NH == 1) or (aligner == "bowtie2" and AS > XS and read1_first if flag & 0x40 or not paired else read2_first)): if chr in region_loci: for region, loci in region_loci[chr].items(): region = region.split('-')[0].lower() _, _, loci_left, loci_right = loci # there might be a different candidate region for each of left and right reads if pos >= loci_left and pos < loci_right: extract_read.add(region) break if block_size > 0: chr_region_num = "%s-%d" % (chr, pos / block_size) whole_extract_read.add(chr_region_num) if flag & 0x40 or not paired: # left read read1_first = False if not read1: if flag & 0x10: # reverse complement read1 = [typing_common.reverse_complement(read), qual[::-1]] else: read1 = [read, qual] else: assert flag & 0x80 # right read read2_first = False if flag & 0x10: # reverse complement read2 = [typing_common.reverse_complement(read), qual[::-1]] else: read2 = [read, qual] for region in extract_read: write_read(gzip_dic[region][0], prev_read_name, read1[0], read1[1]) if paired: write_read(gzip_dic[region][1], prev_read_name, read2[0], read2[1]) for chr_region_num in whole_extract_read: region_chr, region_num = chr_region_num.split('-') region_num = int(region_num) if region_chr not in whole_gzip_dic: continue assert region_num < len(whole_gzip_dic[region_chr]) write_read(whole_gzip_dic[region_chr][region_num][0], prev_read_name, read1[0], read1[1]) if paired: write_read(whole_gzip_dic[region_chr][region_num][1], prev_read_name, read2[0], read2[1]) for gzip1_proc, gzip2_proc in gzip_dic.values(): gzip1_proc.stdin.close() if paired: gzip2_proc.stdin.close() for gzip_list in whole_gzip_dic.values(): for gzip1_proc, gzip2_proc in gzip_list: gzip1_proc.stdin.close() if paired: gzip2_proc.stdin.close() if threads <= 1: work(fq_fname_base, fq_fname, fq_fname2, ranges, simulation, verbose) else: parallel_work(pids, work, fq_fname_base, fq_fname, fq_fname2, ranges, simulation, verbose) if threads > 1: wait_pids(pids)
def genotype(base_fname, target_region_list, fastq, read_fnames, alignment_fname, threads, num_editdist, assembly, local_database, verbose, debug): # variants, backbone sequence, and other sequeces genotype_fnames = ["%s.fa" % base_fname, "%s.locus" % base_fname, "%s.snp" % base_fname, "%s.index.snp" % base_fname, "%s.haplotype" % base_fname, "%s.link" % base_fname, "%s.coord" % base_fname, "%s.clnsig" % base_fname] # hisat2 graph index files genotype_fnames += ["%s.%d.ht2" % (base_fname, i+1) for i in range(8)] if not typing_common.check_files(genotype_fnames): print("Error: some of the following files are missing!", file=sys.stderr) for fname in genotype_fnames: print("\t%s" % fname, file=sys.stderr) sys.exit(1) # Read region alleles (names and sequences) regions = {} region_loci = {} for line in open("%s.locus" % base_fname): family, allele_name, chr, left, right = line.strip().split()[:5] family = family.lower() if len(target_region_list) > 0 \ and family not in target_region_list: continue locus_name = allele_name.split('*')[0] if family in target_region_list \ and len(target_region_list[family]) > 0 \ and locus_name not in target_region_list[family]: continue left = int(left) right = int(right) if family not in region_loci: region_loci[family] = [] region_loci[family].append([locus_name, allele_name, chr, left, right]) if len(region_loci) <= 0: print("Warning: no region exists!", file=sys.stderr) sys.exit(1) # Align reads, and sort the alignments into a BAM file if len(read_fnames) > 0: alignment_fname = align_reads(base_fname, read_fnames, fastq, threads, verbose) assert alignment_fname != "" and os.path.exists(alignment_fname) if not os.path.exists(alignment_fname + ".bai"): index_bam(alignment_fname, verbose) assert os.path.exists(alignment_fname + ".bai") # Extract reads and perform genotyping for family, loci in region_loci.items(): print("Analyzing %s ..." % family.upper(), file=sys.stderr) for locus_name, allele_name, chr, left, right in loci: out_read_fname = "%s.%s" % (family, locus_name) if verbose: print("\tExtracting reads beloning to %s-%s ..." \ % (family, locus_name), file=sys.stderr) extracted_read_fnames = extract_reads(alignment_fname, chr, left, right, out_read_fname, len(read_fnames) != 1, # paired? fastq, verbose) perform_genotyping(base_fname, family, [locus_name], extracted_read_fnames, fastq, num_editdist, assembly, local_database, threads, verbose, debug) print("\n", file=sys.stderr)
def build_genotype_genome(base_fname, inter_gap, intra_gap, threads, database_list, use_clinvar, use_commonvar, verbose): # Download HISAT2 index HISAT2_fnames = ["grch38", "genome.fa", "genome.fa.fai"] if not typing_common.check_files(HISAT2_fnames): typing_common.download_genome_and_index() # Load genomic sequences chr_dic, chr_names, chr_full_names = typing_common.read_genome(open("genome.fa")) genotype_vars, genotype_haplotypes, genotype_clnsig = {}, {}, {} if use_clinvar: # Extract variants from the ClinVar database CLINVAR_fnames = ["clinvar.vcf.gz", "clinvar.snp", "clinvar.haplotype", "clinvar.clnsig"] if not typing_common.check_files(CLINVAR_fnames): if not os.path.exists("clinvar.vcf.gz"): os.system("wget ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/archive/2017/clinvar_20170404.vcf.gz") assert os.path.exists("clinvar.vcf.gz") extract_cmd = ["hisat2_extract_snps_haplotypes_VCF.py"] extract_cmd += ["--inter-gap", str(inter_gap), "--intra-gap", str(intra_gap), "--genotype-vcf", "clinvar.vcf.gz", "genome.fa", "/dev/null", "clinvar"] if verbose: print >> sys.stderr, "\tRunning:", ' '.join(extract_cmd) proc = subprocess.Popen(extract_cmd, stdout=open("/dev/null", 'w'), stderr=open("/dev/null", 'w')) proc.communicate() if not typing_common.check_files(CLINVAR_fnames): print >> sys.stderr, "Error: extract variants from clinvar failed!" sys.exit(1) # Read variants to be genotyped genotype_vars = typing_common.read_variants("clinvar.snp") # Read haplotypes genotype_haplotypes = typing_common.read_haplotypes("clinvar.haplotype") # Read information about clinical significance genotype_clnsig = typing_common.read_clnsig("clinvar.clnsig") if use_commonvar: # Extract variants from dbSNP database commonvar_fbase = "snp144Common" commonvar_fnames = ["%s.snp" % commonvar_fbase, "%s.haplotype" % commonvar_fbase] if not typing_common.check_files(commonvar_fnames): if not os.path.exists("%s.txt.gz" % commonvar_fbase): os.system("wget http://hgdownload.cse.ucsc.edu/goldenPath/hg38/database/%s.txt.gz" % commonvar_fbase) assert os.path.exists("%s.txt.gz" % commonvar_fbase) os.system("gzip -cd %s.txt.gz | awk 'BEGIN{OFS=\"\t\"} {if($2 ~ /^chr/) {$2 = substr($2, 4)}; if($2 == \"M\") {$2 = \"MT\"} print}' > %s.txt" % (commonvar_fbase, commonvar_fbase)) extract_cmd = ["hisat2_extract_snps_haplotypes_UCSC.py", "--inter-gap", str(inter_gap), "--intra-gap", str(intra_gap), "genome.fa", "%s.txt" % commonvar_fbase, commonvar_fbase] if verbose: print >> sys.stderr, "\tRunning:", ' '.join(extract_cmd) proc = subprocess.Popen(extract_cmd, stdout=open("/dev/null", 'w'), stderr=open("/dev/null", 'w')) proc.communicate() if not typing_common.check_files(commonvar_fnames): print >> sys.stderr, "Error: extract variants from clinvar failed!" sys.exit(1) # Read variants to be genotyped genotype_vars = typing_common.read_variants("%s.snp" % commonvar_fbase) # Read haplotypes genotype_haplotypes = typing_common.read_haplotypes("%s.haplotype" % commonvar_fbase) # Genes to be genotyped genotype_genes = {} # Read genes or genomics regions for database_name in database_list: # Extract HLA variants, backbone sequence, and other sequeces typing_common.extract_database_if_not_exists(database_name, [], # locus_list inter_gap, intra_gap, True, # partial? verbose) locus_fname = "%s.locus" % database_name assert os.path.exists(locus_fname) for line in open(locus_fname): HLA_name, chr, left, right, length, exon_str, strand = line.strip().split() left, right = int(left), int(right) length = int(length) if chr not in chr_names: continue if chr not in genotype_genes: genotype_genes[chr] = [] genotype_genes[chr].append([left, right, length, HLA_name, database_name, exon_str, strand]) # Write genotype genome var_num, haplotype_num = 0, 0 genome_out_file = open("%s.fa" % base_fname, 'w') locus_out_file = open("%s.locus" % base_fname, 'w') var_out_file = open("%s.snp" % base_fname, 'w') index_var_out_file = open("%s.index.snp" % base_fname, 'w') haplotype_out_file = open("%s.haplotype" % base_fname, 'w') link_out_file = open("%s.link" % base_fname, 'w') coord_out_file = open("%s.coord" % base_fname, 'w') clnsig_out_file = open("%s.clnsig" % base_fname, 'w') for c in range(len(chr_names)): chr = chr_names[c] chr_full_name = chr_full_names[c] assert chr in chr_dic chr_seq = chr_dic[chr] chr_len = len(chr_seq) if chr in genotype_genes: chr_genes = genotype_genes[chr] def gene_cmp(a, b): a_left, a_right, a_length = a[:3] b_left, b_right, b_length = b[:3] if a_left != b_left: return a_left - b_left if a_right != b_right: return a_right - b_right return a_lenght - b_length chr_genes = sorted(chr_genes, cmp=gene_cmp) else: chr_genes = [] chr_genotype_vars, chr_genotype_vari = [], 0 if chr in genotype_vars: chr_genotype_vars = genotype_vars[chr] chr_genotype_haplotypes, chr_genotype_hti = [], 0 if chr in genotype_haplotypes: chr_genotype_haplotypes = genotype_haplotypes[chr] def add_vars(left, right, chr_genotype_vari, chr_genotype_hti, haplotype_num): # Output variants with clinical significance while chr_genotype_vari < len(chr_genotype_vars): var_left, var_type, var_data, var_id = chr_genotype_vars[chr_genotype_vari] var_right = var_left if var_type == "deletion": var_right += var_data if var_right > right: break if var_right >= left: chr_genotype_vari += 1 continue out_str = "%s\t%s\t%s\t%d\t%s" % (var_id, var_type, chr, var_left + off, var_data) print >> var_out_file, out_str print >> index_var_out_file, out_str if var_id in genotype_clnsig: var_gene, clnsig = genotype_clnsig[var_id] print >> clnsig_out_file, "%s\t%s\t%s" % \ (var_id, var_gene, clnsig) chr_genotype_vari += 1 # Output haplotypes while chr_genotype_hti < len(chr_genotype_haplotypes): ht_left, ht_right, ht_vars = chr_genotype_haplotypes[chr_genotype_hti] if ht_right > right: break if ht_right >= left: chr_genotype_hti += 1 continue print >> haplotype_out_file, "ht%d\t%s\t%d\t%d\t%s" % \ (haplotype_num, chr, ht_left + off, ht_right + off, ','.join(ht_vars)) chr_genotype_hti += 1 haplotype_num += 1 return chr_genotype_vari, chr_genotype_hti, haplotype_num out_chr_seq = "" off = 0 prev_right = 0 for gene in chr_genes: left, right, length, name, family, exon_str, strand = gene chr_genotype_vari, chr_genotype_hti, haplotype_num = add_vars(left, right, chr_genotype_vari, chr_genotype_hti, haplotype_num) # Read HLA backbone sequences allele_seqs = typing_common.read_allele_sequences("%s_backbone.fa" % family) # Read HLA variants allele_vars = typing_common.read_variants("%s.snp" % family) allele_index_vars = typing_common.read_variants("%s.index.snp" % family) # Read HLA haplotypes allele_haplotypes = typing_common.read_haplotypes("%s.haplotype" % family) # Read HLA link information between haplotypes and variants links = typing_common.read_links("%s.link" % family) if name not in allele_seqs or \ name not in allele_vars or \ name not in allele_haplotypes: continue allele_seq = allele_seqs[name] vars, index_vars = allele_vars[name], allele_index_vars[name] index_var_ids = set() for _, _, _, var_id in index_vars: index_var_ids.add(var_id) haplotypes = allele_haplotypes[name] assert length == len(allele_seq) assert left < chr_len and right < chr_len # Skipping overlapping genes if left < prev_right: print >> sys.stderr, "Warning: skipping %s ..." % (name) continue varID2htID = {} assert left < right prev_length = right - left + 1 assert prev_length <= length if prev_right < left: out_chr_seq += chr_seq[prev_right:left] # Output gene (genotype_genome.gene) print >> locus_out_file, "%s\t%s\t%s\t%d\t%d\t%s\t%s" % \ (family.upper(), name, chr, len(out_chr_seq), len(out_chr_seq) + length - 1, exon_str, strand) # Output coord (genotype_genome.coord) print >> coord_out_file, "%s\t%d\t%d\t%d" % \ (chr, len(out_chr_seq), left, right - left + 1) out_chr_seq += allele_seq # Output variants (genotype_genome.snp and genotype_genome.index.snp) for var in vars: var_left, var_type, var_data, var_id = var new_var_id = "hv%d" % var_num varID2htID[var_id] = new_var_id new_var_left = var_left + left + off assert var_type in ["single", "deletion", "insertion"] assert new_var_left < len(out_chr_seq) if var_type == "single": assert out_chr_seq[new_var_left] != var_data elif var_type == "deletion": assert new_var_left + var_data <= len(out_chr_seq) else: assert var_type == "insertion" out_str = "%s\t%s\t%s\t%d\t%s" % (new_var_id, var_type, chr, new_var_left, var_data) print >> var_out_file, out_str if var_id in index_var_ids: print >> index_var_out_file, out_str var_num += 1 # Output haplotypes (genotype_genome.haplotype) for haplotype in haplotypes: ht_left, ht_right, ht_vars = haplotype new_ht_left = ht_left + left + off assert new_ht_left < len(out_chr_seq) new_ht_right = ht_right + left + off assert new_ht_left <= new_ht_right assert new_ht_right <= len(out_chr_seq) new_ht_vars = [] for var_id in ht_vars: assert var_id in varID2htID new_ht_vars.append(varID2htID[var_id]) print >> haplotype_out_file, "ht%d\t%s\t%d\t%d\t%s" % \ (haplotype_num, chr, new_ht_left, new_ht_right, ','.join(new_ht_vars)) haplotype_num += 1 # Output link information between alleles and variants (genotype_genome.link) for link in links: var_id, allele_names = link if var_id not in varID2htID: continue new_var_id = varID2htID[var_id] print >> link_out_file, "%s\t%s" % (new_var_id, allele_names) off += (length - prev_length) prev_right = right + 1 # Write the rest of the Vars chr_genotype_vari, chr_genotype_hti, haplotype_num = add_vars(sys.maxint, sys.maxint, chr_genotype_vari, chr_genotype_hti, haplotype_num) print >> coord_out_file, "%s\t%d\t%d\t%d" % \ (chr, len(out_chr_seq), prev_right, len(chr_seq) - prev_right) out_chr_seq += chr_seq[prev_right:] assert len(out_chr_seq) == len(chr_seq) + off # Output chromosome sequence print >> genome_out_file, ">%s" % (chr_full_name) line_width = 60 for s in range(0, len(out_chr_seq), line_width): print >> genome_out_file, out_chr_seq[s:s+line_width] genome_out_file.close() locus_out_file.close() var_out_file.close() index_var_out_file.close() haplotype_out_file.close() link_out_file.close() coord_out_file.close() clnsig_out_file.close() partial_out_file = open("%s.partial" % base_fname, 'w') for database in database_list: for line in open("%s.partial" % database): allele_name = line.strip() print >> partial_out_file, "%s\t%s" % (database.upper(), allele_name) partial_out_file.close() # Index genotype_genome.fa index_cmd = ["samtools", "faidx", "%s.fa" % base_fname] subprocess.call(index_cmd) # Build HISAT-genotype graph indexes based on the above information hisat2_index_fnames = ["%s.%d.ht2" % (base_fname, i+1) for i in range(8)] build_cmd = ["hisat2-build", "-p", str(threads), "--snp", "%s.index.snp" % base_fname, "--haplotype", "%s.haplotype" % base_fname, "%s.fa" % base_fname, "%s" % base_fname] if verbose: print >> sys.stderr, "\tRunning:", ' '.join(build_cmd) subprocess.call(build_cmd, stdout=open("/dev/null", 'w'), stderr=open("/dev/null", 'w')) if not typing_common.check_files(hisat2_index_fnames): print >> sys.stderr, "Error: indexing failed! Perhaps, you may have forgotten to build hisat2 executables?" sys.exit(1)
def extract_reads(base_fname, database_list, read_dir, out_dir, suffix, read_fname, fastq, paired, simulation, threads, max_sample, job_range, verbose): genotype_fnames = ["%s.fa" % base_fname, "%s.locus" % base_fname, "%s.snp" % base_fname, "%s.haplotype" % base_fname, "%s.link" % base_fname, "%s.coord" % base_fname, "%s.clnsig" % base_fname] # hisat2 graph index files genotype_fnames += ["%s.%d.ht2" % (base_fname, i+1) for i in range(8)] if not typing_common.check_files(genotype_fnames): print >> sys.stderr, "Error: %s related files do not exist as follows:" % base_fname for fname in genotype_fnames: print >> sys.stderr, "\t%s" % fname sys.exit(1) filter_region = len(database_list) > 0 ranges = [] regions, region_loci = {}, {} for line in open("%s.locus" % base_fname): family, allele_name, chr, left, right = line.strip().split()[:5] if filter_region and family.lower() not in database_list: continue region_name = "%s-%s" % (family, allele_name.split('*')[0]) assert region_name not in regions regions[region_name] = allele_name left, right = int(left), int(right) """ exons = [] for exon in exon_str.split(','): exon_left, exon_right = exon.split('-') exons.append([int(exon_left), int(exon_right)]) """ if chr not in region_loci: region_loci[chr] = {} region_loci[chr][region_name] = [allele_name, chr, left, right] database_list.add(family.lower()) if out_dir != "" and not os.path.exists(out_dir): os.mkdir(out_dir) # Extract reads if len(read_fname) > 0: if paired: fq_fnames = [read_fname[0]] fq_fnames2 = [read_fname[1]] else: fq_fnames = read_fname else: if paired: fq_fnames = glob.glob("%s/*.1.%s" % (read_dir, suffix)) else: fq_fnames = glob.glob("%s/*.%s" % (read_dir, suffix)) count = 0 pids = [0 for i in range(threads)] for file_i in range(len(fq_fnames)): if file_i >= max_sample: break fq_fname = fq_fnames[file_i] if job_range[1] > 1: if job_range[0] != (file_i % job_range[1]): continue fq_fname_base = fq_fname.split('/')[-1] one_suffix = ".1." + suffix if fq_fname_base.find(one_suffix) != -1: fq_fname_base = fq_fname_base[:fq_fname_base.find(one_suffix)] else: fq_fname_base = fq_fname_base.split('.')[0] if paired: if read_dir == "": fq_fname2 = fq_fnames2[file_i] else: fq_fname2 = "%s/%s.2.%s" % (read_dir, fq_fname_base, suffix) if not os.path.exists(fq_fname2): print >> sys.stderr, "%s does not exist." % fq_fname2 continue else: fq_fname2 = "" if paired: if out_dir != "": if os.path.exists("%s/%s.extracted.1.fq.gz" % (out_dir, fq_fname_base)): continue else: if out_dir != "": if os.path.exists("%s/%s.extracted.fq.gz" % (out_dir, fq_fname_base)): continue count += 1 print >> sys.stderr, "\t%d: Extracting reads from %s" % (count, fq_fname_base) def work(fq_fname_base, fq_fname, fq_fname2, ranges, simulation, verbose): aligner_cmd = ["hisat2"] if not fastq: aligner_cmd += ["-f"] aligner_cmd += ["-x", base_fname] aligner_cmd += ["--no-spliced-alignment", "--max-altstried", "64"] if paired: aligner_cmd += ["-1", fq_fname, "-2", fq_fname2] else: aligner_cmd += ["-U", fq_fname] if verbose: print >> sys.stderr, "\t\trunning", ' '.join(aligner_cmd) align_proc = subprocess.Popen(aligner_cmd, stdout=subprocess.PIPE, stderr=open("/dev/null", 'w')) gzip_dic = {} out_dir_slash = out_dir if out_dir != "": out_dir_slash += "/" for database in database_list: if paired: # LP6005041-DNA_A01.extracted.1.fq.gz gzip1_proc = subprocess.Popen(["gzip"], stdin=subprocess.PIPE, stdout=open("%s%s.%s.extracted.1.fq.gz" % (out_dir_slash, fq_fname_base, database), 'w'), stderr=open("/dev/null", 'w')) # LP6005041-DNA_A01.extracted.2.fq.gz gzip2_proc = subprocess.Popen(["gzip"], stdin=subprocess.PIPE, stdout=open("%s%s.%s.extracted.2.fq.gz" % (out_dir_slash, fq_fname_base, database), 'w'), stderr=open("/dev/null", 'w')) else: # LP6005041-DNA_A01.extracted.fq.gz gzip1_proc = subprocess.Popen(["gzip"], stdin=subprocess.PIPE, stdout=open("%s%s.%s.extracted.fq.gz" % (out_dir_slash, fq_fname_base, database), 'w'), stderr=open("/dev/null", 'w')) gzip_dic[database] = [gzip1_proc, gzip2_proc if paired else None] def write_read(gzip_proc, read_name, seq, qual): if fastq: gzip_proc.stdin.write("@%s\n" % read_name) gzip_proc.stdin.write("%s\n" % seq) gzip_proc.stdin.write("+\n") gzip_proc.stdin.write("%s\n" % qual) else: gzip_proc.stdin.write(">%s\n" % prev_read_name) gzip_proc.stdin.write("%s\n" % seq) prev_read_name, extract_read, read1, read2 = "", False, [], [] for line in align_proc.stdout: if line.startswith('@'): continue line = line.strip() cols = line.split() read_name, flag, chr, pos, mapQ, cigar, _, _, _, read, qual = cols[:11] flag, pos = int(flag), int(pos) strand = '-' if flag & 0x10 else '+' AS, NH = "", "" for i in range(11, len(cols)): col = cols[i] if col.startswith("AS"): AS = int(col[5:]) elif col.startswith("NH"): NH = int(col[5:]) if (not simulation and read_name != prev_read_name) or \ (simulation and read_name.split('|')[0] != prev_read_name.split('|')[0]): if extract_read: write_read(gzip_dic[region][0], prev_read_name, read1[0], read1[1]) if paired: write_read(gzip_dic[region][1], prev_read_name, read2[0], read2[1]) prev_read_name, extract_read, read1, read2 = read_name, False, [], [] if flag & 0x4 == 0 and NH == 1 and chr in region_loci: for region, loci in region_loci[chr].items(): region = region.split('-')[0].lower() _, _, loci_left, loci_right = loci if pos >= loci_left and pos < loci_right: extract_read = True break if flag & 0x40 or not paired: # left read if not read1: if flag & 0x10: # reverse complement read1 = [typing_common.reverse_complement(read), qual[::-1]] else: read1 = [read, qual] else: assert flag & 0x80 # right read if flag & 0x10: # reverse complement read2 = [typing_common.reverse_complement(read), qual[::-1]] else: read2 = [read, qual] if extract_read: write_read(gzip_dic[region][0], prev_read_name, read1[0], read1[1]) if paired: write_read(gzip_dic[region][1], prev_read_name, read2[0], read2[1]) for gzip1_proc, gzip2_proc in gzip_dic.values(): gzip1_proc.stdin.close() if paired: gzip2_proc.stdin.close() if threads <= 1: work(fq_fname_base, fq_fname, fq_fname2, ranges, simulation, verbose) else: parallel_work(pids, work, fq_fname_base, fq_fname, fq_fname2, ranges, simulation, verbose) if threads > 1: wait_pids(pids)
def extract_reads(base_fname, database_list, read_dir, out_dir, suffix, read_fname, fastq, paired, simulation, threads, max_sample, job_range, verbose): genotype_fnames = [ "%s.fa" % base_fname, "%s.locus" % base_fname, "%s.snp" % base_fname, "%s.haplotype" % base_fname, "%s.link" % base_fname, "%s.coord" % base_fname, "%s.clnsig" % base_fname ] # hisat2 graph index files genotype_fnames += ["%s.%d.ht2" % (base_fname, i + 1) for i in range(8)] if not typing_common.check_files(genotype_fnames): print >> sys.stderr, "Error: %s related files do not exist as follows:" % base_fname for fname in genotype_fnames: print >> sys.stderr, "\t%s" % fname sys.exit(1) filter_region = len(database_list) > 0 ranges = [] regions, region_loci = {}, {} for line in open("%s.locus" % base_fname): family, allele_name, chr, left, right = line.strip().split()[:5] if filter_region and family.lower() not in database_list: continue region_name = "%s-%s" % (family, allele_name.split('*')[0]) assert region_name not in regions regions[region_name] = allele_name left, right = int(left), int(right) """ exons = [] for exon in exon_str.split(','): exon_left, exon_right = exon.split('-') exons.append([int(exon_left), int(exon_right)]) """ if chr not in region_loci: region_loci[chr] = {} region_loci[chr][region_name] = [allele_name, chr, left, right] database_list.add(family.lower()) if out_dir != "" and not os.path.exists(out_dir): os.mkdir(out_dir) # Extract reads if len(read_fname) > 0: if paired: fq_fnames = [read_fname[0]] fq_fnames2 = [read_fname[1]] else: fq_fnames = read_fname else: if paired: fq_fnames = glob.glob("%s/*.1.%s" % (read_dir, suffix)) else: fq_fnames = glob.glob("%s/*.%s" % (read_dir, suffix)) count = 0 pids = [0 for i in range(threads)] for file_i in range(len(fq_fnames)): if file_i >= max_sample: break fq_fname = fq_fnames[file_i] if job_range[1] > 1: if job_range[0] != (file_i % job_range[1]): continue fq_fname_base = fq_fname.split('/')[-1] one_suffix = ".1." + suffix if fq_fname_base.find(one_suffix) != -1: fq_fname_base = fq_fname_base[:fq_fname_base.find(one_suffix)] else: fq_fname_base = fq_fname_base.split('.')[0] if paired: if read_dir == "": fq_fname2 = fq_fnames2[file_i] else: fq_fname2 = "%s/%s.2.%s" % (read_dir, fq_fname_base, suffix) if not os.path.exists(fq_fname2): print >> sys.stderr, "%s does not exist." % fq_fname2 continue else: fq_fname2 = "" if paired: if out_dir != "": if os.path.exists("%s/%s.extracted.1.fq.gz" % (out_dir, fq_fname_base)): continue else: if out_dir != "": if os.path.exists("%s/%s.extracted.fq.gz" % (out_dir, fq_fname_base)): continue count += 1 print >> sys.stderr, "\t%d: Extracting reads from %s" % (count, fq_fname_base) def work(fq_fname_base, fq_fname, fq_fname2, ranges, simulation, verbose): aligner_cmd = ["hisat2"] if not fastq: aligner_cmd += ["-f"] aligner_cmd += ["-x", base_fname] aligner_cmd += ["--no-spliced-alignment", "--max-altstried", "64"] if paired: aligner_cmd += ["-1", fq_fname, "-2", fq_fname2] else: aligner_cmd += ["-U", fq_fname] if verbose: print >> sys.stderr, "\t\trunning", ' '.join(aligner_cmd) align_proc = subprocess.Popen(aligner_cmd, stdout=subprocess.PIPE, stderr=open("/dev/null", 'w')) gzip_dic = {} out_dir_slash = out_dir if out_dir != "": out_dir_slash += "/" for database in database_list: if paired: # LP6005041-DNA_A01.extracted.1.fq.gz gzip1_proc = subprocess.Popen( ["gzip"], stdin=subprocess.PIPE, stdout=open( "%s%s.%s.extracted.1.fq.gz" % (out_dir_slash, fq_fname_base, database), 'w'), stderr=open("/dev/null", 'w')) # LP6005041-DNA_A01.extracted.2.fq.gz gzip2_proc = subprocess.Popen( ["gzip"], stdin=subprocess.PIPE, stdout=open( "%s%s.%s.extracted.2.fq.gz" % (out_dir_slash, fq_fname_base, database), 'w'), stderr=open("/dev/null", 'w')) else: # LP6005041-DNA_A01.extracted.fq.gz gzip1_proc = subprocess.Popen( ["gzip"], stdin=subprocess.PIPE, stdout=open( "%s%s.%s.extracted.fq.gz" % (out_dir_slash, fq_fname_base, database), 'w'), stderr=open("/dev/null", 'w')) gzip_dic[database] = [ gzip1_proc, gzip2_proc if paired else None ] def write_read(gzip_proc, read_name, seq, qual): if fastq: gzip_proc.stdin.write("@%s\n" % read_name) gzip_proc.stdin.write("%s\n" % seq) gzip_proc.stdin.write("+\n") gzip_proc.stdin.write("%s\n" % qual) else: gzip_proc.stdin.write(">%s\n" % prev_read_name) gzip_proc.stdin.write("%s\n" % seq) prev_read_name, extract_read, read1, read2 = "", False, [], [] for line in align_proc.stdout: if line.startswith('@'): continue line = line.strip() cols = line.split() read_name, flag, chr, pos, mapQ, cigar, _, _, _, read, qual = cols[: 11] flag, pos = int(flag), int(pos) strand = '-' if flag & 0x10 else '+' AS, NH = "", "" for i in range(11, len(cols)): col = cols[i] if col.startswith("AS"): AS = int(col[5:]) elif col.startswith("NH"): NH = int(col[5:]) if (not simulation and read_name != prev_read_name) or \ (simulation and read_name.split('|')[0] != prev_read_name.split('|')[0]): if extract_read: write_read(gzip_dic[region][0], prev_read_name, read1[0], read1[1]) if paired: write_read(gzip_dic[region][1], prev_read_name, read2[0], read2[1]) prev_read_name, extract_read, read1, read2 = read_name, False, [], [] if flag & 0x4 == 0 and NH == 1 and chr in region_loci: for region, loci in region_loci[chr].items(): region = region.split('-')[0].lower() _, _, loci_left, loci_right = loci if pos >= loci_left and pos < loci_right: extract_read = True break if flag & 0x40 or not paired: # left read if not read1: if flag & 0x10: # reverse complement read1 = [ typing_common.reverse_complement(read), qual[::-1] ] else: read1 = [read, qual] else: assert flag & 0x80 # right read if flag & 0x10: # reverse complement read2 = [ typing_common.reverse_complement(read), qual[::-1] ] else: read2 = [read, qual] if extract_read: write_read(gzip_dic[region][0], prev_read_name, read1[0], read1[1]) if paired: write_read(gzip_dic[region][1], prev_read_name, read2[0], read2[1]) for gzip1_proc, gzip2_proc in gzip_dic.values(): gzip1_proc.stdin.close() if paired: gzip2_proc.stdin.close() if threads <= 1: work(fq_fname_base, fq_fname, fq_fname2, ranges, simulation, verbose) else: parallel_work(pids, work, fq_fname_base, fq_fname, fq_fname2, ranges, simulation, verbose) if threads > 1: wait_pids(pids)
def genotype(base_fname, target_region_list, fastq, read_fnames, alignment_fname, threads, num_editdist, assembly, local_database, verbose, debug): # variants, backbone sequence, and other sequeces genotype_fnames = ["%s.fa" % base_fname, "%s.locus" % base_fname, "%s.snp" % base_fname, "%s.index.snp" % base_fname, "%s.haplotype" % base_fname, "%s.link" % base_fname, "%s.coord" % base_fname, "%s.clnsig" % base_fname] # hisat2 graph index files genotype_fnames += ["%s.%d.ht2" % (base_fname, i+1) for i in range(8)] if not typing_common.check_files(genotype_fnames): print >> sys.stderr, "Error: some of the following files are missing!" for fname in genotype_fnames: print >> sys.stderr, "\t%s" % fname sys.exit(1) # Read region alleles (names and sequences) regions, region_loci = {}, {} for line in open("%s.locus" % base_fname): family, allele_name, chr, left, right = line.strip().split()[:5] family = family.lower() if len(target_region_list) > 0 and \ family not in target_region_list: continue locus_name = allele_name.split('*')[0] if family in target_region_list and \ len(target_region_list[family]) > 0 and \ locus_name not in target_region_list[family]: continue left, right = int(left), int(right) if family not in region_loci: region_loci[family] = [] region_loci[family].append([locus_name, allele_name, chr, left, right]) if len(region_loci) <= 0: print >> sys.stderr, "Warning: no region exists!" sys.exit(1) # Align reads, and sort the alignments into a BAM file if len(read_fnames) > 0: alignment_fname = align_reads(base_fname, read_fnames, fastq, threads, verbose) assert alignment_fname != "" and os.path.exists(alignment_fname) if not os.path.exists(alignment_fname + ".bai"): index_bam(alignment_fname, verbose) assert os.path.exists(alignment_fname + ".bai") # Extract reads and perform genotyping for family, loci in region_loci.items(): print >> sys.stderr, "Analyzing %s ..." % family.upper() for locus_name, allele_name, chr, left, right in loci: out_read_fname = "%s.%s" % (family, locus_name) if verbose: print >> sys.stderr, "\tExtracting reads beloning to %s-%s ..." % \ (family, locus_name) extracted_read_fnames = extract_reads(alignment_fname, chr, left, right, out_read_fname, len(read_fnames) != 1, # paired? fastq, verbose) perform_genotyping(base_fname, family, [locus_name], extracted_read_fnames, fastq, num_editdist, assembly, local_database, threads, verbose) print >> sys.stderr