def calculateExonIntrons(entrez_genes, bedfile, column_index, chroms, fragment_size, totalcount, out_file): lib_name = (bedfile).split('/')[-1] # remove directory suffix = lib_name.split('.')[-1] # txt lib_name = lib_name.split('.')[0] extension = "-" + lib_name +'.' + suffix +"1" if Utility_extended.fileExists(bedfile): if Utility_extended.chrom_files_exist(chroms, extension) != 1: # Separate by chrom and sort by start print chroms, extension, " files do not exist, separate by chroms. " Utility_extended.separate_by_chrom_sort(chroms, bedfile, extension, [column_index]) else: print bedfile, " is not found"; sys.exit(1) all_reads_on_shared_exons = {} # {entrezID:[((start, end), read_count)]} all_reads_on_shared_introns = {} # {entrezID:[((start, end), read_count)]} all_reads_on_merged_transcripts = {} #{entrezID:[((start, end), read_count)]} all_summary = {} for chrom in chroms: chrombed = chrom + extension entrez_genes_by_chrom = Entrez.KnownEntrezGenes([chrom], entrez_genes.subset_by_chrom(chrom)) (reads_on_shared_exons, reads_on_shared_introns, reads_on_merged_transcripts, summary) = calculateExonIntrons_by_chrom (entrez_genes_by_chrom, chrombed, fragment_size, totalcount, out_file) #if chrom == chroms[0]: #myid = reads_on_shared_exons.keys()[0] #test(entrez_genes_by_chrom, reads_on_shared_introns, myid) all_reads_on_shared_exons.update(reads_on_shared_exons) all_reads_on_shared_introns.update(reads_on_shared_introns) all_reads_on_merged_transcripts.update(reads_on_merged_transcripts) all_summary.update(summary) SeparateByChrom.cleanup(chroms, extension) return (all_reads_on_shared_exons, all_reads_on_shared_introns, all_reads_on_merged_transcripts, summary)
def Calculate3UTRUsage(entrez_genes, bedfile, chroms, outfile, threshold, PAfile, extension, index): """ entrez genes are made sure to be on one strand, the bed file are reads for that strand entrez_genes is a KnownEntrezGenes class object The raw read file needs to conform to bed format column_index: column in bed file for sorting """ # Separate reads by chrom rawreadslibName1 = (bedfile).split('/')[-1] rawreadssuffix1 = rawreadslibName1.split('.')[-1] rawreadslibName1 = rawreadslibName1.split('.')[0] rawreadsextension1 = "-" + rawreadslibName1 + '.' + rawreadssuffix1 + "1" if Utility_extended.fileExists(bedfile): if Utility_extended.chrom_files_exist(chroms, rawreadsextension1) != 1: # Separate by chrom and sort by start print chroms, rawreadsextension1, " files do not exist, separate by chroms and sort each file according to the second column. " Utility_extended.separate_by_chrom_sort(chroms, bedfile, rawreadsextension1, str(index)) else: print bedfile, " is not found" sys.exit(1) #This part is to access the polyadenylation sites PA1 = open(PAfile, 'r') PAsiteslist = [] PA2 = 'i' while PA2 != '': PA2 = PA1.readline() if PA2 != '': PA3 = PA2.strip('\n') PA4 = PA3.split('\t') PAsiteslist.append((PA4[0], PA4[1])) PA1.close() # Here the output is 'a', i.e. the output is appended to an existing file instead of creating one outf = open(outfile, 'a') for chrom in chroms: if chrom in entrez_genes.chroms: # a KnownEntrezGenes object entrez_genes_by_chrom = Entrez.KnownEntrezGenes( [chrom], entrez_genes.subset_by_chrom(chrom)) # Get the read locations if Utility_extended.fileExists(chrom + rawreadsextension1): f = open(chrom + rawreadsextension1, 'r') tag_positions = [] for line in f: line = line.strip() sline = line.split() #make sure the extension is always 0, otherwise the rest of the program might not work as intended tag_positions.append( associate_tags_with_regions.tag_position(sline, 0)) f.close() if not Utility_extended.is_list_sorted(tag_positions): tag_positions.sort() #By this point tag_positions is a sorted list of all the reads located on the strand and chromosome the code is currently dealing with for entrez_id in entrez_genes_by_chrom.entrez_ids: gene = entrez_genes_by_chrom.entrez_genes[ entrez_id] # an EntrezGene class object # get_3UTRs gets the ENTREZ 3'UTR, which appears to generally give the beginning of the 3'UTR and a site very close to the most distal polyadenylation site three_UTRs = gene.get_3UTRs() # Mastertuplemaker uses the ENTREZ 3'UTR and the polyA sites given to create the true data for the 3'UTR needed for CUTR_vs_AUTR to work true3UTRstarts, true3UTRends, UTRregion_start, UTRregion_end, UTRbeginning = Mastertuplemaker( three_UTRs, PAsiteslist, chrom, gene.strand, extension) #value should always be 1 as only 3'UTR with more than 1 polyA site need be considered if len(true3UTRends) > 1: #find all reads inside the 3'UTR inside_reads = associate_tags_with_3UTR( tag_positions, UTRregion_start, UTRregion_end) #finds reads in each region of the 3'UTR and calculates aUTR/cUTR for each of them #PolyAsites potentially useful for output RUDs, basic_RUD, PolyAsites = CUTR_vs_AUTR( true3UTRstarts, true3UTRends, inside_reads, gene.strand, threshold) #important if one wants to output gene_symbol information gene_symbol = [] for mytranscript in gene.transcripts: if mytranscript.additional_annotations[ 0] not in gene_symbol: gene_symbol.append( mytranscript.additional_annotations[0]) #outline to use to output RUDs outline = str( entrez_id ) + "\t" + chrom + "\t" + gene.strand + "\t" + str( basic_RUD) + "\t" + ",".join(map(str, RUDs)) + "\n" #outline to use to output polyA information for a species #outline = str(entrez_id) + "\t" + chrom + "\t" + gene.strand + "\t" + str(UTRbeginning) + "\t" + ",".join(map(str, PolyAsites)) + "\n" outf.write(outline) outf.close()
def getReadCount(KnownGenes, bedfile, chroms, fragment_size, region_type, upstream_extension, downstream_extension, totalcount, out_file): """ Known genes are made sure to be on one strand, and the bed file are reads for that strand The raw read file needs to conform to bed format """ ReadCount = {} # keyed by name, valued by (rc, length, rpkm) # Separate by chrom reads rawreadslibName1 = (bedfile).split('/')[-1] rawreadssuffix1 = rawreadslibName1.split('.')[-1] rawreadslibName1 = rawreadslibName1.split('.')[0] rawreadsextension1 = "-" + rawreadslibName1 + '.' + rawreadssuffix1 + "1" if Utility_extended.fileExists(bedfile): if Utility_extended.chrom_files_exist(chroms, rawreadsextension1) != 1: # Separate by chrom and sort by start print chroms, rawreadsextension1, " files do not exist, separate by chroms and sort each file according to the second column. " Utility_extended.separate_by_chrom_sort(chroms, bedfile, rawreadsextension1, [2]) else: print bedfile, " is not found" sys.exit(1) # dictionary has chrom as key and ucsc_lite object (name, chrom, strand, txStart, txEnd) as values if region_type == 'Promoter': region_dic = KnownGenes.getPromoters(upstream_extension, downstream_extension) elif region_type == 'GeneBody': region_dic = KnownGenes.getGenebodys(downstream_extension) elif region_type == 'ExtendedGeneBody': region_dic = KnownGenes.getExtendedGenebodys(upstream_extension, downstream_extension) elif region_type == 'PromoterGenebody': region_dic = KnownGenes.getPromotergenebodys(upstream_extension) elif region_type == 'GeneEnd': region_dic = KnownGenes.getGeneEnds(upstream_extension, downstream_extension) elif region_type == 'ExonicRegion': region_dic = KnownGenes.getExons() elif region_type == 'IntronicRegion': region_dic = KnownGenes.getIntrons() elif region_type == '5UTR': region_dic = KnownGenes.get5UTRs(upstream_extension, downstream_extension) elif region_type == '3UTR': region_dic = KnownGenes.get3UTRs(upstream_extension, downstream_extension) else: print region_type, "is not recognized" exit(1) outf = open(out_file, 'a') for chrom in chroms: chrombed = chrom + rawreadsextension1 if Utility_extended.fileExists(chrombed) and (chrom in KnownGenes.keys()): tag_position_list = [] inf = open(chrombed, 'r') for line in inf: if not re.match("#", line): line = line.strip() sline = line.split() tag_position_list.append( associate_tags_with_regions.tag_position( sline, fragment_size)) inf.close() if Utility_extended.is_list_sorted(tag_position_list) != 1: tag_position_list.sort() if len(region_dic[chrom]) > 0: for region in region_dic[chrom]: thisregion = [(region.txStart, region.txEnd)] (total_length, rc) = get_read_count_on_regions(thisregion, tag_position_list) if total_length > 0: RPKM = rc * (1000.0 / total_length) * ( 1000000 / float(totalcount)) else: assert rc < 0.01 RPKM = 0 outline = str(region.name) + '\t' + str(rc) + '\t' + str( total_length) + '\t' + str(RPKM) + '\n' outf.write(outline) ReadCount[region.name] = (rc, total_length, RPKM) outf.close() #SeparateByChrom.cleanup(chroms, rawreadsextension1) return ReadCount
def get_read_count_on_onic_transcript(KnownGenes, bedfile, chroms, fragment_size, region_type, totalcount, out_file): """ Return: a dictionary keyed by geneName valued by TotalReadCount,TotalLength, RPKM """ ReadCount = {} # keyed by name, valued by (rc, length, rpkm) # Separate by chrom reads rawreadslibName1 = (bedfile).split('/')[-1] rawreadssuffix1 = rawreadslibName1.split('.')[-1] rawreadslibName1 = rawreadslibName1.split('.')[0] rawreadsextension1 = "-" + rawreadslibName1 + '.' + rawreadssuffix1 + "1" if Utility_extended.fileExists(bedfile): if Utility_extended.chrom_files_exist(chroms, rawreadsextension1) != 1: # Separate by chrom and sort by start print chroms, rawreadsextension1, " files do not exist, separate by chroms and sort each file according to the second column. " Utility_extended.separate_by_chrom_sort(chroms, bedfile, rawreadsextension1, [2]) # sort by start else: print bedfile, " is not found" sys.exit(1) outf = open(out_file, 'a') for chrom in chroms: chrombed = chrom + rawreadsextension1 if Utility_extended.fileExists(chrombed) and (chrom in KnownGenes.keys()): tag_position_list = [] inf = open(chrombed, 'r') for line in inf: if not re.match("#", line): line = line.strip() sline = line.split() tag_position_list.append( associate_tags_with_regions.tag_position( sline, fragment_size)) inf.close() if Utility_extended.is_list_sorted(tag_position_list) != 1: tag_position_list.sort() for gene in KnownGenes[chrom]: if region_type == "ExonicTranscript": ons = gene.getExons() elif region_type == "IntronicTranscript": ons = gene.getIntrons() else: print region_type, "is not recognized." exit(1) if len(ons > 0): (total_length, rc) = get_read_count_on_regions(ons, tag_position_list) RPKM = rc * (1000.0 / total_length) * (1000000 / float(totalcount)) else: total_length = 0 rc = 0 RPKM = 0 outline = str(gene.name) + '\t' + str(rc) + '\t' + str( total_length) + '\t' + str(RPKM) + '\n' outf.write(outline) ReadCount[region.name] = (rc, total_length, RPKM) outf.close() #SeparateByChrom.cleanup(chroms, rawreadsextension1) return ReadCount
def Calculate3UTRUsage(entrez_genes, bedfile, column_index, chroms, fragment_size, downstream_extension, outfile): """ entrez genes are made sure to be on one strand, the bed file are reads for that strand entrez_genes is a KnownEntrezGenes class object The raw read file needs to conform to bed format column_index: column in bed file for sorting """ # Separate reads by chrom rawreadslibName1 = (bedfile).split('/')[-1] rawreadssuffix1 = rawreadslibName1.split('.')[-1] rawreadslibName1 = rawreadslibName1.split('.')[0] rawreadsextension1 = "-" + rawreadslibName1 + '.' + rawreadssuffix1 + "1" if Utility_extended.fileExists(bedfile): if Utility_extended.chrom_files_exist(chroms, rawreadsextension1) != 1: # Separate by chrom and sort by start print chroms, rawreadsextension1, " files do not exist, separate by chroms and sort each file according to the second column. " Utility_extended.separate_by_chrom_sort(chroms, bedfile, rawreadsextension1, [column_index]) else: print bedfile, " is not found" sys.exit(1) # Here the output is 'a' outf = open(outfile, 'a') for chrom in chroms: if chrom in entrez_genes.chroms: # a KnownEntrezGenes object entrez_genes_by_chrom = Entrez.KnownEntrezGenes( [chrom], entrez_genes.subset_by_chrom(chrom)) # this_chrom_length = chrom_lengths[chrom] # Get the read locations if Utility_extended.fileExists(chrom + rawreadsextension1): f = open(chrom + rawreadsextension1, 'r') tag_positions = [] for line in f: line = line.strip() sline = line.split() tag_positions.append( associate_tags_with_regions.tag_position( sline, fragment_size)) if not Utility_extended.is_list_sorted(tag_positions): tag_positions.sort() f.close() for entrez_id in entrez_genes_by_chrom.entrez_ids: gene = entrez_genes_by_chrom.entrez_genes[ entrez_id] # an EntrezGene class object three_UTRs = gene.get_3UTRs(downstream_extension) print three_UTRs union = Utility_extended.union( three_UTRs ) # Find the union of 3UTRs [(start, end)], returns a [(start,end)] if len(union) > 1: print "There are disjoint 3UTRs in %s" % ( str(entrez_id)) else: # returns [((start, end), [tag_positions])], [tag_positions] = return[0][1] inside_reads = (Utility_extended. associate_simple_tags_with_regions( tag_positions, union))[0][1] total_read_count = len(inside_reads) RUD = CUTR_vs_AUTR(three_UTRs, inside_reads, gene.strand) ## For the set of genes, use the distal 3UTR at the designated representative 3UTR #myindex = Calculate3UTRUsageIndexFromCuratedGenes.find_distal_3UTR(genes) #gene = genes[myindex] #results = ThreeUTRCharacteristics(gene, inside_reads) gene_symbol = [] for mytranscript in gene.transcripts: if mytranscript.additional_annotations[ 0] not in gene_symbol: gene_symbol.append( mytranscript.additional_annotations[0]) union_length = union[0][1] - union[0][0] + 1 outline = str(entrez_id) + "\t" + str( union_length) + "\t" + str(RUD) + "\t" + str( total_read_count) + "\t" + ','.join([ transcript.name for transcript in gene.transcripts ]) + "\t" + ','.join(gene_symbol) + "\n" outf.write(outline) outf.close()
def calculateExonIntrons(entrez_genes, bedfile, column_index, chroms, fragment_size, totalcount, out_file=None): """ entrez_genes is a EntrezGenes class object. The core is a entrez_genes.entrez_genes is a dic (keyed by entrez_id) of lists of EntrezGene object return: all_reads_on_shared_exons = {} # {entrezID:[((start, end), read_count)]} all_reads_on_shared_introns = {} # {entrezID:[((start, end), read_count)]} all_reads_on_merged_transcripts = {} #{entrezID:[((start, end), read_count)]} all_summary = {} # {entrezID:{attribute:value}} (summary[entrez_id])["merged_exons_rc"] = merged_exons_rc (summary[entrez_id])["merged_exon_RPKM"] = merged_exon_RPKM (summary[entrez_id])["merged_exons_total_length"] = merged_exons_total_length (summary[entrez_id])["shared_exons_rc"] = shared_exons_rc (summary[entrez_id])["shared_exon_RPKM"] = shared_exon_RPKM (summary[entrez_id])["shared_exons_total_length"] = shared_exons_total_length (summary[entrez_id])["shared_introns_rc"] = shared_introns_rc (summary[entrez_id])["shared_intron_RPKM"] = shared_intron_RPKM (summary[entrez_id])["shared_introns_total_length"] = shared_introns_total_length (summary[entrez_id])["merged_transcript_rc"] = merged_transcript_rc (summary[entrez_id])["merged_transcript_RPKM"] = merged_transcript_RPKM (summary[entrez_id])["merged_transcript_length"] = merged_transcript_length """ lib_name = (bedfile).split('/')[-1] # remove directory suffix = lib_name.split('.')[-1] # txt lib_name = lib_name.split('.')[0] extension = "-" + lib_name + '.' + suffix + "1" if Utility_extended.fileExists(bedfile): if Utility_extended.chrom_files_exist(chroms, extension) != 1: # Separate by chrom and sort by start print chroms, extension, " files do not exist, separate by chroms. " Utility_extended.separate_by_chrom_sort(chroms, bedfile, extension, [column_index]) else: print bedfile, " is not found" sys.exit(1) all_reads_on_shared_exons = {} # {entrezID:[((start, end), read_count)]} all_reads_on_shared_introns = {} # {entrezID:[((start, end), read_count)]} all_reads_on_merged_transcripts = { } #{entrezID:[((start, end), read_count)]} all_summary = {} # {entrezID:{attributes}} for chrom in chroms: chrombed = chrom + extension if chrom in entrez_genes.chroms: entrez_genes_by_chrom = Entrez.KnownEntrezGenes( [chrom], entrez_genes.subset_by_chrom(chrom)) (reads_on_shared_exons, reads_on_shared_introns, reads_on_merged_transcripts, summary) = calculateExonIntrons_by_chrom(entrez_genes_by_chrom, chrombed, fragment_size, totalcount, out_file) #if chrom == chroms[0]: #myid = reads_on_shared_exons.keys()[0] #test(entrez_genes_by_chrom, reads_on_shared_introns, myid) all_reads_on_shared_exons.update(reads_on_shared_exons) all_reads_on_shared_introns.update(reads_on_shared_introns) all_reads_on_merged_transcripts.update(reads_on_merged_transcripts) all_summary.update(summary) print len(all_summary.keys()) SeparateByChrom.cleanup(chroms, extension) return (all_reads_on_shared_exons, all_reads_on_shared_introns, all_reads_on_merged_transcripts, all_summary)
def Calculate3UTRUsage(entrez_genes, bedfile, chroms, outfile, threshold, PAfile, extension, index): """ entrez genes are made sure to be on one strand, the bed file are reads for that strand entrez_genes is a KnownEntrezGenes class object The raw read file needs to conform to bed format column_index: column in bed file for sorting """ # Separate reads by chrom rawreadslibName1 = (bedfile).split('/')[-1] rawreadssuffix1 = rawreadslibName1.split('.')[-1] rawreadslibName1 = rawreadslibName1.split('.')[0] rawreadsextension1 = "-" + rawreadslibName1 +'.' + rawreadssuffix1 + "1" if Utility_extended.fileExists(bedfile): if Utility_extended.chrom_files_exist(chroms, rawreadsextension1) != 1: # Separate by chrom and sort by start print chroms, rawreadsextension1, " files do not exist, separate by chroms and sort each file according to the second column. " Utility_extended.separate_by_chrom_sort(chroms, bedfile, rawreadsextension1, str(index)) else: print bedfile, " is not found" sys.exit(1) #This part is to access the polyadenylation sites PA1 = open(PAfile, 'r') PAsiteslist = [] PA2 = 'i' while PA2 != '': PA2 = PA1.readline() if PA2 != '': PA3 = PA2.strip('\n') PA4 = PA3.split('\t') PAsiteslist.append((PA4[0],PA4[1])) PA1.close() # Here the output is 'a', i.e. the output is appended to an existing file instead of creating one outf = open(outfile, 'a') for chrom in chroms: if chrom in entrez_genes.chroms: # a KnownEntrezGenes object entrez_genes_by_chrom = Entrez.KnownEntrezGenes([chrom], entrez_genes.subset_by_chrom(chrom)) # Get the read locations if Utility_extended.fileExists(chrom + rawreadsextension1): f = open(chrom + rawreadsextension1, 'r') tag_positions = [] for line in f: line = line.strip() sline = line.split() #make sure the extension is always 0, otherwise the rest of the program might not work as intended tag_positions.append(associate_tags_with_regions.tag_position(sline, 0)) f.close() if not Utility_extended.is_list_sorted(tag_positions): tag_positions.sort() #By this point tag_positions is a sorted list of all the reads located on the strand and chromosome the code is currently dealing with for entrez_id in entrez_genes_by_chrom.entrez_ids: gene = entrez_genes_by_chrom.entrez_genes[entrez_id] # an EntrezGene class object # get_3UTRs gets the ENTREZ 3'UTR, which appears to generally give the beginning of the 3'UTR and a site very close to the most distal polyadenylation site three_UTRs = gene.get_3UTRs() # Mastertuplemaker uses the ENTREZ 3'UTR and the polyA sites given to create the true data for the 3'UTR needed for CUTR_vs_AUTR to work true3UTRstarts, true3UTRends, UTRregion_start, UTRregion_end, UTRbeginning = Mastertuplemaker(three_UTRs,PAsiteslist,chrom,gene.strand, extension) #value should always be 1 as only 3'UTR with more than 1 polyA site need be considered if len(true3UTRends) > 1: #find all reads inside the 3'UTR inside_reads = associate_tags_with_3UTR(tag_positions, UTRregion_start, UTRregion_end) #finds reads in each region of the 3'UTR and calculates aUTR/cUTR for each of them #PolyAsites potentially useful for output RUDs, basic_RUD, PolyAsites = CUTR_vs_AUTR(true3UTRstarts, true3UTRends, inside_reads, gene.strand, threshold) #important if one wants to output gene_symbol information gene_symbol = [] for mytranscript in gene.transcripts: if mytranscript.additional_annotations[0] not in gene_symbol: gene_symbol.append(mytranscript.additional_annotations[0]) #outline to use to output RUDs outline = str(entrez_id) + "\t" + chrom + "\t" + gene.strand + "\t" + str(basic_RUD) + "\t" + ",".join(map(str, RUDs)) + "\n" #outline to use to output polyA information for a species #outline = str(entrez_id) + "\t" + chrom + "\t" + gene.strand + "\t" + str(UTRbeginning) + "\t" + ",".join(map(str, PolyAsites)) + "\n" outf.write(outline) outf.close()