def main():
## using argparser to load arguments from workflow
parser = argparse.ArgumentParser()
parser.add_argument('--trspdictfilestring',
help='input transcript density files')
parser.add_argument('--UTRfilestring', help='UTRs file')
parser.add_argument('--cdsDenThresh',
help='boolean value- should density filter be used?')
parser.add_argument(
'--norm_type', help='type of normalizaion, should be raw reads or rpm')
parser.add_argument('--raw_dense_thresh',
help='threshold for CDS density for raw normalization')
parser.add_argument('--rpm_dense_thresh',
help='threshold for CDS density for rpm normalization')
parser.add_argument(
'--inset_choice',
help='inset values to be used to avoid start and stop codon peaks')
parser.add_argument('--outfilestring', help='output file name')
# parser.add_argument
args = parser.parse_args()
args.cdsDenThresh = args.cdsDenThresh == 'True'
print "ARGS: ", args.cdsDenThresh
# print bool(args.cdsDenThresh)
trspdict = rph.readcountsf(args.trspdictfilestring)
UTRdict = rph.readindict(open(args.UTRfilestring, "rU"))
countsIDlist, countsOutdict = build_count_tables(
trspdict, UTRdict, args.inset_choice, args.cdsDenThresh,
args.norm_type, float(args.raw_dense_thresh),
float(args.rpm_dense_thresh), args.outfilestring)
write_countTable_to_csv(countsIDlist, countsOutdict, args.outfilestring)
def main():
## using argparser to load arguments from workflow
parser = argparse.ArgumentParser()
parser.add_argument('--trspdictfilestring',
help='input transcript density files')
parser.add_argument('--UTRfilestring', help='UTRs file')
parser.add_argument('--cdsDenThresh',
help='boolean value- should density filter be used?')
parser.add_argument(
'--norm_type', help='type of normalizaion, should be raw reads or rpm')
parser.add_argument('--raw_dense_thresh',
help='threshold for CDS density for raw normalization')
parser.add_argument('--rpm_dense_thresh',
help='threshold for CDS density for rpm normalization')
parser.add_argument(
'--inset_choice',
help='inset values to be used to avoid start and stop codon peaks')
parser.add_argument('--outfilestring', help='output file name')
parser.add_argument(
'--totreads',
help='total number of reads used after raw densebuilder run')
parser.add_argument(
'--stopcodons',
help=
'csv file with positions of all stopcodons, riboseq_stopcodon_finder.py'
)
# parser.add_argument
args = parser.parse_args()
# utr3adj = pd.read_csv(stopframe_csv, index_col=0)
utr3adj = pd.read_csv(args.stopcodons, index_col=0)
# print utr3adj.head()
# print utr3adj.loc['ENST00000426362.6']
# print utr3adj.loc[utr3adj['#transcript'] == 'ENST00000426362.6']
trspdict = rph.readcountsf(args.trspdictfilestring)
UTRdict = rph.readindict(open(args.UTRfilestring, "rU"))
# print UTRdict['ENST00000426362.6']
countsIDlist, countsOutdict = build_count_tables(
trspdict, UTRdict, utr3adj, args.inset_choice, bool(args.cdsDenThresh),
args.norm_type, float(args.raw_dense_thresh),
float(args.rpm_dense_thresh), args.outfilestring, int(args.totreads),
minUtr3len)
write_countTable_to_csv(countsIDlist, countsOutdict, args.outfilestring)
def codonaverage(self):
outlist, headers, motiffilelist = [], [], []
headers.append("motif")
for motif in self.motifs:
motiffile = args.motiffilerootpath + motif + "_1.csv"
motiffilelist.append(motiffile)
headers.append(motif)
outlist.append(headers)
codon_occu = []
codon_occu.append(self.sample_name)
f_output = open(args.outfileparams, "w")
f_output.write("Density file is " + str(self.sample_name) + "\n")
f_output.write("cds5trim is " + str(args.cds5trim) + "\n")
f_output.write("cds3trim is " + str(args.cds3trim) + "\n")
f_output.write("Seqwin is " + str(args.seqwin) + "\n")
f_output.write("Motiflist is " + str(motiffilelist) + "\n")
readcountsdict = rph.readcountsf(args.trspdictfilestring)
exclusionmodule = exclusionfiles[0]
if exclusionmodule != '0':
exclusiondict = self.readindict(open(exclusionmodule, "rU"))
else:
exclusiondict = '0'
print "Exclusion file is " + str(exclusionmodule)
UTRdict = rph.readindict(open(args.UTRfilestring, "rU"))
occupancy = self.occupancy(readcountsdict, motiffilelist,
exclusiondict, codon_occu, UTRdict,
f_output)
outlist.append(codon_occu)
f_output.close()
co = np.asarray(outlist) # convert outlist to a np.array
output = co.T
# print "output: ", output
# print "self.outlistfile: ", self.outlistfile
self.writerows(output, self.outlistfile) # write these rows to a csv
comments += "Threshold signifies minimal rpkm needed in coding region for gene to be in the average.\n"
comments += "alignpos =1 anchors average around the start codon and only includes 5'UTRs. alignpos =2 is the same for stop codon."
fc = open(args.outfilebase + "_" + str(args.alignpos) + "_output.txt", "w")
fc.write(comments)
fc.write("\n")
fc.write("Avggene was called with parameters:\n")
fc.write("transcripts= " + str(args.trspdictfilestring) + "\n")
fc.write("filtermodule= " + str(args.filtermodule) + "\n")
fc.write("exclusionmodule= " + str(args.exclusionmodule) + "\n")
fc.write("threshold= " + str(args.threshold) + "\n")
fc.write("regionlength5= " + str(args.regionlength5) + "\n")
fc.write("regionlength3= " + str(args.regionlength3) + "\n")
fc.write("equalweight= " + str(args.equalweight) + "\n")
fc.write("alignpos= " + str(args.alignpos) + "\n")
fc.close()
if args.filtermodule != '0':
filterdict = rph.readindict(open(args.filtermodule, "rU"))
else:
filterdict = '0'
if args.exclusionmodule != '0':
exclusiondict = rph.readindict(open(args.exclusionmodule, "rU"))
else:
exclusiondict = '0'
trspdict = rph.readcountsf(args.trspdictfilestring)
UTRdict = rph.readindict(open(args.UTRfilestring, "rU"))
metagene = Avggene(args.regionlength5, args.regionlength3, trspdict,
UTRdict, filterdict, exclusiondict, args.threshold,
args.alignpos, args.equalweight, args.outfilebase)
metagene.totalavg()
def build_count_table_single(sample, UTRdict = UTRdict,):
"""
This is the function that returns total counts within the following regions:
mRNA, cds, utr5, utr3
*** Do not filter out any transcripts in output file ***
Length of each region is modified by insets to account for biases around start and stop codons
incorporating trspdict into this function
### using totreads hack to get back to unnormalized, integer values without going thru the trouble of also mapping RAW A site reads
"""
densitystring = "Density_rpm"
ribosome_shift = "A"
fp_assign_path = '%s/FPassignment/%s/%s/%s' % (rootpath, genome_name, experiment, sample)
totreads_countfile = "%s/%s_FPassigned_counts.txt" % (fp_assign_path, sample)
totreadcountf = open(totreads_countfile, "r")
totreads = int(totreadcountf.read())
totreadcountf.close()
trspdictfilestring = '%s/%s/density5p_rnaseq/%s/%sf_' %(
fp_assign_path, densitystring, sample, sample)
outfolder = "%s/countTables" % (fp_assign_path) # make a new folder to store count tables
if not os.path.exists(outfolder): os.makedirs(outfolder)
### load the counts for all transcripts into a dictionary
trspdict= rph.readcountsf(trspdictfilestring)
### define the insets, and select which insets to use when assigning footprint counts
### purpose of insets is to avoid counting ribosomes around the start and stop codons
defaultInsets = { 'utr5Inset3' : 6, 'cdsInset5' : 18, 'cdsInset3' : 15, 'utr3Inset5' : 6 }
zeroInsets = { 'utr5Inset3' : 0, 'cdsInset5' : 0, 'cdsInset3' : 0, 'utr3Inset5' : 0 }
customInsets = { 'utr5Inset3' : 15, 'cdsInset5' : 24, 'cdsInset3' : 15, 'utr3Inset5' : 15 }
if inset_choice == "default":
insets = defaultInsets
elif inset_choice == "zero":
insets = zeroInsets
elif inset_choice == "custom":
insets = customInsets
else:
print "Insets were not set"
sys.exit()
countsIDlist = []
countsOutdict = {}
noUTRentry = 0
zeroUtrlen = 0
zeroCdsdense = 0
totalCountedTranscripts = 0
for trsp in trspdict:
if UTRdict.has_key(trsp)!=True: # check to make sure density file has an annotation in the UTR csv
noUTRentry +=1
continue
### Schema for subsetting UTRdict is below:
# from csv: #transcript,chrom,featnum,strand,mrna_len,cds_len,5utr_len,3utr_len,gene_name,stopcodon,stop4nt
# position in list: key, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9
# define base region sizes from UTRdict
mrnalen = int(UTRdict[trsp][3])
cdslen = int(UTRdict[trsp][4])
utr5len = int(UTRdict[trsp][5])
utr3len = int(UTRdict[trsp][6])
## count transcripts with zero lengths for utr's
if utr5len == 0 or utr3len == 0:
zeroUtrlen +=1
# print("transcript has zero utr5 len %s") % trsp
# sys.exit()
# if utr3len == 0:
# zeroUtrlen +=1
### get counts from density file
exonsplicedcounts = trspdict[trsp]
### set starts and ends of the coding sequence
cdsstart = utr5len
cdsend = len(exonsplicedcounts) - utr3len
if cdsstart == cdsend:
print "Error, gene length is 0 for transcript %s" % trsp
sys.exit()
### modify region lengths using insets:
## not sure if these modified lengths are necessarily informative here since I am not calcing densities
utr5len = utr5len-insets['utr5Inset3']
cdslen = cdslen-insets['cdsInset5']-insets['cdsInset3']
utr3len = utr3len-insets['utr3Inset5']
mrnalen = utr5len+cdslen+utr3len
### This is the place to add insets for counts
## assigning RAW counts to a given region
utr5Counts = sum(exonsplicedcounts[:cdsstart-insets['utr5Inset3']])
utr3Counts = sum(exonsplicedcounts[cdsend+insets['utr3Inset5']:])
cdsCounts = sum(exonsplicedcounts[cdsstart+insets['cdsInset5']:cdsend-insets['cdsInset3']])
mrnaCounts = int(utr5Counts+cdsCounts+utr3Counts)
### 181208 - adding raw counts back to data tables, use totreads to adjust these back to the appropriate value
RAWutr5Counts = int(utr5Counts*(totreads/1E6))
RAWutr3Counts = int(utr3Counts*(totreads/1E6))
RAWcdsCounts = int(cdsCounts*(totreads/1E6))
RAWmrnaCounts = RAWutr5Counts+RAWcdsCounts+RAWutr3Counts ## nevermind, already had a calculation for RAWcounts, this is not necessary anymore...
## although, this could be a problem that I am filtering out certain transcripts still
## I'm not sure If I can just correct this to be unfiltered...
### define variables to output for final list
transcriptOutlist = [RAWmrnaCounts, RAWcdsCounts, RAWutr5Counts, RAWutr3Counts] #adding row for utr3 adjust:
countsIDlist.append(transcriptOutlist[0]) # contains list of keys for the dictionary
countsOutdict[trsp] = transcriptOutlist # add this transcript to the dictionary
totalCountedTranscripts += 1
print "***********************************************************"
print "Count table for complete for %s" % sample
print "Number of transcripts absent in UTRfile: %s" % noUTRentry
print "Number of transcripts with zero UTR lengths: %s" % zeroUtrlen
print "Number of transcripts with zero CDS density: %s" % zeroCdsdense
print "Total transcripts included in count table: %s" % totalCountedTranscripts
print "***********************************************************"
#### need to think about writing up these things here, might not be strictly necessary
# this is all for writing to the output text file
# fc = open(outfilestring+"_output.txt", "w") # open in appending mode
# fc.write("---Summary of DenseTables Run--- \n")
# fc.write(str(datetime.now())+"\n")
# fc.write("Total reads in library used for normalization: %s \n" % totreads)
# fc.write("Number of transcripts absent in UTRfile: %s \n" % noUTRentry)
# fc.write("Number of transcripts with zero UTR lengths: %s \n" % zeroUtrlen)
# fc.write("Number of transcripts with zero CDS density: %s \n" % zeroCdsdense)
# fc.write("Number of transcripts below min 3'UTR length to first inframe stop: %s \n" % tooShortUtr3AdjLenCalc)
# fc.write("Total transcripts included in count table: %s \n" % totalCountedTranscripts)
# fc.write("%s insets were used with the following values: %s \n" % (inset_choice, insets))
# fc.write("Normalization type: %s \n" % norm_type)
# if cdsDenThresh == True and norm_type == 'raw':
# fc.write("CDS density threshold on raw reads: %s \n" % str(raw_dense_thresh))
# elif cdsDenThresh == True and norm_type == 'rpm':
# fc.write("CDS density threshold on rpm: %s \n" % str(rpm_dense_thresh))
# elif cdsDenThresh == False:
# fc.write("No threshold was set on cds density \n")
# else:
# print "cdsDenThresh not set!"
# fc.close()
# sys.exit()
# fc.close()
# print countsOutdict['uc001qop.2'] # gapdh
dfout = pd.DataFrame.from_dict(countsOutdict, orient='index') #upgrade to pandas v 0.23 to use the columns keyword
cols= [sample+'_mRNACounts', sample+'_cdsCounts', sample+'_utr5Counts', sample+'_utr3Counts']
dfout.columns = cols
# dfout.rename(columns=cols)
# print dfout.head()
df_mRNA = dfout.iloc[:,0]
df_cds = dfout.iloc[:,1]
df_utr5 = dfout.iloc[:,2]
df_utr3 = dfout.iloc[:,3]
return df_mRNA, df_cds, df_utr5, df_utr3
def build_avggene_firstStop(UTRdict, threshold, samp, region_choice,
inset_choice):
"""
frame_selection == ['frameMinusOne', 'frameZero', 'framePlusOne']
"""
fp_assign_path = '%s/FPassignment/%s/%s/%s' % (rootpath, genome_name,
experiment, samp)
trspfilestring = '%s/%s/density%sp_%sshift_%sto%s/%s_%sto%sf/%s_%sto%sf_' % (
fp_assign_path, densitystring, assignment, ribosome_shift, minlen,
maxlen, samp, minlen, maxlen, samp, minlen, maxlen)
totreads_countfile = "%s/%s_FPassigned_counts.txt" % (fp_assign_path, samp)
totreadcountf = open(totreads_countfile, "r")
totreads = int(totreadcountf.read())
totreadcountf.close()
### This is where all the count files are loaded into a dictionary
trspdict = rph.readcountsf(trspfilestring) ### this takes a minute
## create a list of 0's that is the length of the region of interest
## add 3 to account for the stop codon
## this will be added to for every transcript
averagegene = [0 for num in range(0, 3 + upstreamNTs + downstreamNTs)
] # add 3 for start or stop codon
# print averagegene
## add counters and set to zero
noUTRentry = 0 ### discard transcripts not in UTRdict
zeroCdsdense = 0 ### discard transcripts with zero reads in CDS
lowCdsdense = 0 ## Optional CDS density thresholding, in RPKM
regionTooShort = 0 ### not enough 3'UTR in region of interest past first stop codon
totalCountedTranscripts = 0 ## number included in final output
### calculate mRNA-region densities
defaultInsets = {
'utr5Inset3': 6,
'cdsInset5': 18,
'cdsInset3': 15,
'utr3Inset5': 6
}
zeroInsets = {
'utr5Inset3': 0,
'cdsInset5': 0,
'cdsInset3': 0,
'utr3Inset5': 0
}
customInsets = {
'utr5Inset3': 15,
'cdsInset5': 24,
'cdsInset3': 15,
'utr3Inset5': 15
}
if inset_choice == "default":
insets = defaultInsets
elif inset_choice == "zero":
insets = zeroInsets
elif inset_choice == "custom":
insets = customInsets
else:
print "Insets were not set"
sys.exit()
### Iterated through transcripts one at a time, retrieving counts in region of interest:
for trsp in trspdict:
# if trsp != 'ENST00000319248.13': ## testing with PRDX1
# continue
### Load in count file for the transcript here
exonsplicedcounts = trspdict[trsp]
if UTRdict.has_key(
trsp
) != True: # check to make sure density file has an annotation in the UTR csv
noUTRentry += 1
continue
mrnalen = int(UTRdict[trsp][3])
cdslen = int(UTRdict[trsp][4])
utr5len = int(UTRdict[trsp][5])
utr3len = int(UTRdict[trsp][6])
assert mrnalen == cdslen + utr5len + utr3len ## check that this is true
### define Coding sequence here
cdsstart = utr5len
cdsend = len(
exonsplicedcounts) - utr3len # cdsend == first position of utr3
if cdsstart == cdsend:
print "Error, gene length is 0 for transcript " + trsp
sys.exit()
# cdscounts= exonsplicedcounts[cdsstart:cdsend] ### counts are already in rpm
# utr3CountList = exonsplicedcounts[cdsend:]
# cdsdensity= sum(cdscounts)/len(cdscounts) ### unmodified CDS density
# if cdsdensity == 0:
# zeroCdsdense += 1
# continue
# if cdsdensity*float(1000)< int(threshold): # Threshold on cds density: (thresholding on "rpkm")
# lowCdsdense += 1
# continue
### Calculate Region Densities ###
utr5lenMod = utr5len - insets['utr5Inset3']
cdslenMod = cdslen - insets['cdsInset5'] - insets['cdsInset3']
utr3lenMod = utr3len - insets['utr3Inset5']
mrnalenMod = utr5lenMod + cdslenMod + utr3lenMod
utr5Counts = sum(exonsplicedcounts[:cdsstart - insets['utr5Inset3']])
cdsCounts = sum(
exonsplicedcounts[cdsstart + insets['cdsInset5']:cdsend -
insets['cdsInset3']])
utr3Counts = sum(exonsplicedcounts[cdsend + insets['utr3Inset5']:])
mrnaCounts = utr5Counts + cdsCounts + utr3Counts
### RAW counts
RAWutr5Counts = int(utr5Counts * (totreads / 1E6))
RAWutr3Counts = int(utr3Counts * (totreads / 1E6))
RAWcdsCounts = int(cdsCounts * (totreads / 1E6))
RAWmrnaCounts = int(mrnaCounts * (totreads / 1E6))
### denisites
# mrnaDensity = (mrnaCounts/mrnalenMod)
cdsDensity = (cdsCounts / cdslenMod)
# utr5Density = (utr5Counts/utr5lenMod)
# utr3Density = (utr3Counts/utr3lenMod)
#### RPKM densities
# mrnaDensity_rpkm = (mrnaCounts/mrnalenMod) * 1000
cdsDensity_rpkm = (cdsCounts / cdslenMod) * 1000
# utr5Density_rpkm = (utr5Counts/utr5lenMod) * 1000
# utr3Density_rpkm = (utr3Counts/utr3lenMod) * 1000
### throw out zero's
if cdsDensity == 0:
zeroCdsdense += 1
continue
if cdsDensity * float(1000) < int(
threshold
): # Threshold on cds density: (thresholding on "rpkm")
lowCdsdense += 1
continue
### define vector in valid CDS region, normalize by cdsDensity
cdsSplicedCounts = exonsplicedcounts[cdsstart +
insets['cdsInset5']:cdsend -
insets['cdsInset3']]
cdsNormCounts = [
rpf / cdsDensity for rpf in cdsSplicedCounts
] ## just region of cds within insets for counts, sum/len == 1
# print cdslenMod, "modified cds length"
# print sum(cdsNormCounts), "total normalized counts, should equal length of cds"
# print sum(cdsNormCounts)/len(cdsNormCounts) ## this should == 1
### define vector for whole transcript, normlaized by cdsDensity
exonNormCounts = [
rpf / cdsDensity for rpf in exonsplicedcounts
] ## counts normalized by cds density, using only region within insets
# print exonNormCounts
# print sum(exonNormCounts)
# print sum(exonNormCounts)/len(exonNormCounts) ## should typically be less than 1, unless greater density from utrs and start/stop codons
### for start codon metagenes -
if region_choice == 'start':
### check boundaries:
# print mRNAdf.loc[trsp]['mRNAseqs'][cdsstart:cdsstart+3] ### start codon
if len(exonNormCounts[cdsstart - upstreamNTs:cdsstart + 3 +
downstreamNTs]
) < upstreamNTs + 3 + downstreamNTs:
regionTooShort += 1
continue
else:
totalCountedTranscripts += 1
avggene_counts = exonNormCounts[cdsstart -
upstreamNTs:cdsstart + 3 +
downstreamNTs]
for i in range(len(avggene_counts)
): ### add these counts to the running total
averagegene[i] += avggene_counts[i]
if region_choice == 'stop':
### check boundaries
# print mRNAdf.loc[trsp]['mRNAseqs'][cdsend-3:cdsend] ### stop codon
if len(exonNormCounts[cdsend - 3 - upstreamNTs:cdsend +
downstreamNTs]
) < upstreamNTs + 3 + downstreamNTs:
regionTooShort += 1
continue
else:
totalCountedTranscripts += 1
# print upstreamNTs+3+downstreamNTs
# print len(exonNormCounts[cdsend-3-upstreamNTs:cdsend+downstreamNTs])
avggene_counts = exonNormCounts[cdsend - 3 -
upstreamNTs:cdsend +
downstreamNTs]
for i in range(len(avggene_counts)
): ### add these counts to the running total
averagegene[i] += avggene_counts[i]
averagegene_equal = [rpf / totalCountedTranscripts for rpf in averagegene
] ### divide by total number of valid transcripts
# print averagegene
# print averagegene_equal
positions = range(-upstreamNTs - 1,
downstreamNTs + 2) # start or stop codon is [-1,0,1]
df = pd.DataFrame({'position': positions, 'avg': averagegene_equal})
df = df[['position', 'avg']]
# print df['avg'].sum()/len(df)
# print df
print "Avggene run compolete for sample %s" % samp
print "Transcripts inculded %s" % totalCountedTranscripts
print "Number of transcripts absent in UTRfile: %s" % noUTRentry
print "Number of transcripts with zero CDS density: %s" % zeroCdsdense
print "Number of transcripts below CDS density threshold: %s" % lowCdsdense
print "Number of transcripts too short avggene region: %s" % regionTooShort
print "- - - - - - - -"
print df
print "- - - - - - - -"
if region_choice == 'start':
alignpos = "1"
elif region_choice == 'stop':
alignpos = "2"
else:
print "alignpos not set!!!"
### write csv file
fp_assign_path = '%s/FPassignment/%s/%s/%s' % (rootpath, genome_name,
experiment, samp)
avggene_csv_path = "%s/avggene%s_ORF%s_%sshift_%s%s150" % (
fp_assign_path, alignpos, norm_type, ribosome_shift, assignment, norm)
if not os.path.exists(avggene_csv_path): os.makedirs(avggene_csv_path)
csv_outfile = "%s/%s_%s_rpkmThresh%s_%sto%sf_avg_%s_cdsNorm.csv" % ( # adding cdsNorm to indicate normalized densities
avggene_csv_path, samp, pop, threshold, minlen, maxlen, alignpos)
df.to_csv(csv_outfile, index=False)
def retrieve_stop_density(din, samp):
d = din.copy()
dropList = []
averagegeneNTC = [0 for num in range(0, 3 + upstreamNTs + downstreamNTs)]
averagegeneTC3 = [0 for num in range(0, 3 + upstreamNTs + downstreamNTs)]
d['NTC_up_counts'] = np.zeros(len(d))
d['NTC_up_dense'] = np.zeros(len(d))
d['NTC_down_counts'] = np.zeros(len(d))
d['NTC_down_dense'] = np.zeros(len(d))
# d['NTC_ratio'] = np.zeros(len(d))
d['TC3_up_counts'] = np.zeros(len(d))
d['TC3_up_dense'] = np.zeros(len(d))
d['TC3_down_counts'] = np.zeros(len(d))
d['TC3_down_dense'] = np.zeros(len(d))
# d['TC3_ratio'] = np.zeros(len(d))
lowCdsdense = 0
totalCountedTranscripts = 0
fp_assign_path = '%s/FPassignment/%s/%s/%s' % (rootpath, genome_name,
experiment, samp)
trspfilestring = '%s/%s/density%sp_%sshift_%sto%s/%s_%sto%sf/%s_%sto%sf_' % (
fp_assign_path, densitystring, assignment, ribosome_shift, minlen,
maxlen, samp, minlen, maxlen, samp, minlen, maxlen)
totreads_countfile = "%s/%s_FPassigned_counts.txt" % (fp_assign_path, samp)
totreadcountf = open(totreads_countfile, "r")
totreads = int(totreadcountf.read())
totreadcountf.close()
defaultInsets = {
'utr5Inset3': 6,
'cdsInset5': 18,
'cdsInset3': 15,
'utr3Inset5': 6
}
zeroInsets = {
'utr5Inset3': 0,
'cdsInset5': 0,
'cdsInset3': 0,
'utr3Inset5': 0
}
customInsets = {
'utr5Inset3': 15,
'cdsInset5': 24,
'cdsInset3': 15,
'utr3Inset5': 15
}
if inset_choice == "default":
insets = defaultInsets
elif inset_choice == "zero":
insets = zeroInsets
elif inset_choice == "custom":
insets = customInsets
else:
print "Insets were not set"
sys.exit()
### This is where all the count files are loaded into a dictionary
trspdict = rph.readcountsf(trspfilestring) ### this takes a minute
for tr in d.index:
# print tr
mrnalen = int(d.loc[tr, 'mrna_len'])
cdslen = int(d.loc[tr, 'cds_len'])
utr5len = int(d.loc[tr, '5utr_len'])
utr3len = int(d.loc[tr, '3utr_len'])
assert mrnalen == cdslen + utr5len + utr3len
exonsplicedcounts = trspdict[tr]
utr3LenAdj = int(
d.loc[tr, 'frameZeroUtr3LenAdj']
) + 3 ## adding +3 here to now include length of stop codon
### calculate normalized counts to cds
cdsstart = utr5len
cdsend = len(
exonsplicedcounts) - utr3len # cdsend == first position of utr3
cdslenMod = cdslen - insets['cdsInset5'] - insets['cdsInset3']
cdsCounts = sum(
exonsplicedcounts[cdsstart + insets['cdsInset5']:cdsend -
insets['cdsInset3']])
cdsDensity = (cdsCounts / cdslenMod)
# print cdsDensity*float(1000)
if cdsDensity * float(1000) < int(
threshold
): # Threshold on cds density: (thresholding on "rpkm")
lowCdsdense += 1
dropList.append(tr)
continue
exonNormCounts = [
rpf / cdsDensity for rpf in exonsplicedcounts
] ## counts normalized by cds density, using only region within insets
# print sum(exonNormCounts)
# print sum(exonNormCounts[cdsstart+insets['cdsInset5']:cdsend-insets['cdsInset3']])
utr3CountList = exonNormCounts[cdsend:]
ntcCounts = exonNormCounts[(cdsend - upstreamNTs - 3):(cdsend +
downstreamNTs)]
tc3Counts = utr3CountList[(utr3LenAdj - upstreamNTs -
3):(utr3LenAdj + downstreamNTs)]
# print len(ntcCounts)
# print ntcCounts
# print len(tc3Counts)
# print tc3Counts
### define regions: upstream counts == [0:27], downstream counts == [36:0]
### more generally: upstream counts == [:upstreamNTs-3], downstream counts == [downstreamNTs+6:]
d.at[tr, 'NTC_up_counts'] = sum(ntcCounts[:upstreamNTs - 3])
d.at[tr, 'NTC_up_dense'] = sum(ntcCounts[:upstreamNTs - 3]) / (
upstreamNTs - 3) ## avoiding codon before stop
d.at[tr, 'NTC_down_counts'] = sum(ntcCounts[downstreamNTs + 6:])
d.at[tr, 'NTC_down_dense'] = sum(ntcCounts[downstreamNTs + 6:]) / (
downstreamNTs - 3) ## avoiding codon after stop
# d.at[tr, 'NTC_ratio'] = d.loc[tr, 'NTC_down_dense']/d.loc[tr, 'NTC_up_dense']
d.at[tr, 'TC3_up_counts'] = sum(tc3Counts[:upstreamNTs - 3])
d.at[tr, 'TC3_up_dense'] = sum(
tc3Counts[:upstreamNTs - 3]) / (upstreamNTs - 3)
d.at[tr, 'TC3_down_counts'] = sum(tc3Counts[downstreamNTs + 6:])
d.at[tr, 'TC3_down_dense'] = sum(
tc3Counts[downstreamNTs + 6:]) / (downstreamNTs - 3)
# d.at[tr, 'TC3_ratio'] = d.loc[tr, 'TC3_down_dense']/d.loc[tr, 'TC3_up_dense']
for i in range(len(ntcCounts)):
averagegeneNTC[i] += ntcCounts[i]
for i in range(len(tc3Counts)):
averagegeneTC3[i] += tc3Counts[i]
totalCountedTranscripts += 1
avgGeneEqualNTC = [
rpf / totalCountedTranscripts for rpf in averagegeneNTC
] ### test average gene plot
avgGeneEqualTC3 = [
rpf / totalCountedTranscripts for rpf in averagegeneTC3
] ### test average gene plot
# print "avgGeneNTC!", avgGeneEqualNTC
# print "avgGeneTC3!", avgGeneEqualTC3
dout = d.drop(dropList, axis=0)
print len(d)
print len(dropList)
print len(dout)
# print exonsplicedcounts
return avgGeneEqualNTC, avgGeneEqualTC3, dout
def region_size_dist_ftsize(readsize, sample):
"""
build the counts for a given region
"""
fp_assign_path = '%s/FPassignment/%s/%s/%s' % (rootpath, genome_name, experiment, sample)
totreads_countfile = "%s/%s_FPassigned_counts.txt" % (fp_assign_path, sample)
totreadcountf = open(totreads_countfile, "r")
totreads = int(totreadcountf.read())
totreadcountf.close()
print "total reads for sample %s = %s" % (sample, totreads)
# for readsize in ftsize:
readsize = str(readsize) # convert to string
trspdictfilestring = '%s/DensityUnnormalized/density5p_0shift_%s/%s_%sf/%s_%sf_' %(
fp_assign_path, readsize, sample, readsize, sample, readsize)
bamfilepath_readsize = '%s/%s_star_default/%s_%s_match.sorted.bam' % (
fp_assign_path, sample, sample, readsize)
bamfile = pysam.AlignmentFile(bamfilepath_readsize, 'rb')
read_count_bam = bamfile.count()
print "total reads in bamfile for sample: %s, read length: %s, equals == %s" % (sample, readsize, read_count_bam)
## check total number of reads in this bamfile:
## build the trspdict now for a given readlength:
trspdict = rph.readcountsf(trspdictfilestring)
## add counters
noUTRentry = 0
zeroUtrlen = 0
zeroUtrlenInsets = 0
zeroCdsdense = 0
lowCdsdense = 0
lowCdsCounts = 0 # adding cds raw read counter
### counters for output
totUtr5Counts = 0
totCdsCounts = 0
totUtr3Counts = 0
totMrnaCounts = 0
## iterate through every transcript in the gtf file
for trsp in trspdict:
if UTRdict.has_key(trsp)!=True: # check to make sure density file has an annotation in the UTR csv
noUTRentry +=1
continue
# from csv: #transcript,chrom,featnum,strand,mrna_len,cds_len,5utr_len,3utr_len,gene_name,stopcodon,stop4nt
# position in list: key, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9
# define base region sizes from UTRdict
mrnalen = int(UTRdict[trsp][3])
cdslen = int(UTRdict[trsp][4])
utr5len = int(UTRdict[trsp][5])
utr3len = int(UTRdict[trsp][6])
### Not sure if I want to keep this here... see how many have lengths of zero first
if utr5len == 0:
zeroUtrlen +=1
# print("transcript has zero utr5 len %s") % trsp
# sys.exit()
continue
if utr3len == 0:
zeroUtrlen +=1
continue
# get counts from density file
exonsplicedcounts = trspdict[trsp]
# set starts and ends
cdsstart = utr5len
cdsend = len(exonsplicedcounts) - utr3len
if cdsstart == cdsend:
print "Error, gene length is 0 for transcript %s" % trsp
sys.exit()
# modify region lengths using insets:
utr5len = utr5len-insets['utr5Inset3']
cdslen = cdslen-insets['cdsInset5']-insets['cdsInset3']
utr3len = utr3len-insets['utr3Inset5']
mrnalen = utr5len+cdslen+utr3len
if utr5len == 0:
zeroUtrlenInsets +=1
# print "transcript has zero utr5 len %s" % trsp
# sys.exit()
continue
if utr3len == 0:
zeroUtrlenInsets +=1
continue
utr5Counts = sum(exonsplicedcounts[:cdsstart-insets['utr5Inset3']])
utr3Counts = sum(exonsplicedcounts[cdsend+insets['utr3Inset5']:])
cdsCounts = sum(exonsplicedcounts[cdsstart+insets['cdsInset5']:cdsend-insets['cdsInset3']])
mrnaCounts = utr5Counts+cdsCounts+utr3Counts
totUtr5Counts += utr5Counts
totCdsCounts += cdsCounts
totUtr3Counts += utr3Counts
totMrnaCounts += mrnaCounts
print "UTR5total = %s, CDStotal = %s, UTR3total = %s, mRNAtotal = %s " % (totUtr5Counts, totCdsCounts, totUtr3Counts, totMrnaCounts)
return [int(readsize), totUtr5Counts, totCdsCounts, totUtr3Counts, totMrnaCounts]
def build_avggene_firstStop(UTRdict, utr3adj, threshold, file,
frame_selection):
"""
frame_selection == ['frameMinusOne', 'frameZero', 'framePlusOne']
Not equally weighted here
"""
fp_assign_path = '%s/FPassignment/%s/%s/%s' % (rootpath, genome_name,
experiment, file)
trspfilestring = '%s/%s/density%sp_%sshift_%sto%s/%s_%sto%sf/%s_%sto%sf_' % (
fp_assign_path, densitystring, assignment, ribosome_shift, minlen,
maxlen, file, minlen, maxlen, file, minlen, maxlen)
totreads_countfile = "%s/%s_FPassigned_counts.txt" % (fp_assign_path, file)
totreadcountf = open(totreads_countfile, "r")
totreads = int(totreadcountf.read())
totreadcountf.close()
### This is where all the count files are loaded into a dictionary
trspdict = rph.readcountsf(trspfilestring) ### this takes a minute
## create a list of 0's that is the length of the region of interest
## add 3 to account for the stop codon
## this will be added to for every transcript
averagegene = [0 for num in range(0, 3 + upstreamNTs + downstreamNTs)
] # add 3 for the inframe stop codon
## add counters and set to zero
noUTRentry = 0 ### discard transcripts not in UTRdict
zeroUtrlen = 0 ### discard transcripts with zero 3'UTR length
zeroCdsdense = 0 ### discard transcripts with zero reads in CDS
lowCdsdense = 0 ## Optional CDS density thresholding, in RPKM
noInframeStops = 0 ### discard transcripts without any inframe stop codons
tooCloseToCDS = 0 ### not enough space between first inframe stop and Coding sequence
utr3tooShort = 0 ### not enough 3'UTR in region of interest past first stop codon
totalCountedTranscripts = 0 ## number included in final output
### Iterated through transcripts one at a time, retrieving counts in region of interest:
for trsp in trspdict:
if UTRdict.has_key(
trsp
) != True: # check to make sure density file has an annotation in the UTR csv
noUTRentry += 1
continue
mrnalen = int(UTRdict[trsp][3])
cdslen = int(UTRdict[trsp][4])
utr5len = int(UTRdict[trsp][5])
utr3len = int(UTRdict[trsp][6])
assert mrnalen == cdslen + utr5len + utr3len ## check that this is true
if utr3len == 0: # only check the 3'UTR here
zeroUtrlen += 1
continue
### set stop selection:
if frame_selection == 'frameZero':
utr3LenAdj = int(
utr3adj.loc[trsp].frameZeroUtr3LenAdj
) + 3 # add plus 3 to include length of stop codon
### utr3LenAdj is now the 0-based position one nucleotide past the first inframe stop codon relative to the 3'UTR
### position of stop codon is utr3CountList[utr3LenAdj-3:utr3LenAdj]
### possibly a stupid way to do this, but ill think about it more
inframeStopCount = int(utr3adj.loc[trsp].frameZeroStopCount)
elif frame_selection == 'framePlusOne':
utr3LenAdj = int(utr3adj.loc[trsp].framePlusOneUtr3LenAdj) + 3
inframeStopCount = int(utr3adj.loc[trsp].framePlusOneStopCount)
elif frame_selection == 'frameMinusOne':
utr3LenAdj = int(utr3adj.loc[trsp].frameMinusOneUtr3LenAdj) + 3
inframeStopCount = int(utr3adj.loc[trsp].frameMinusOneStopCount)
else:
print "frame not set!"
sys.exit()
### filter out trsp's with no inframe stops, or stops too close to CDS or end of transcript
if inframeStopCount == 0:
noInframeStops += 1
continue
if utr3LenAdj - upstreamNTs - 3 < 0: # taking everything up to the stop codon, includes first nt of utr3
tooCloseToCDS += 1
continue
if utr3LenAdj + downstreamNTs > utr3len:
utr3tooShort += 1
continue
### Load in count file for the transcript here
exonsplicedcounts = trspdict[trsp]
### define Coding sequence here
cdsstart = utr5len
cdsend = len(
exonsplicedcounts) - utr3len # cdsend == first position of utr3
if cdsstart == cdsend:
print "Error, gene length is 0 for transcript " + trsp
sys.exit()
cdscounts = exonsplicedcounts[cdsstart:cdsend]
utr3CountList = exonsplicedcounts[cdsend:]
cdsdensity = sum(cdscounts) / len(cdscounts)
if cdsdensity == 0:
zeroCdsdense += 1
continue
if cdsdensity * float(1000) < int(
threshold
): # Threshold on cds density: (thresholding on "rpkm")
lowCdsdense += 1
continue
### create the counts we want for our region
avgGeneCounts = utr3CountList[(utr3LenAdj - upstreamNTs -
3):(utr3LenAdj + downstreamNTs)]
totalCountedTranscripts += 1
for i in range(len(avgGeneCounts)):
averagegene[i] += avgGeneCounts[i]
print "Avggene run compolete for sample %s" % file
print "Frame selection is: %s" % frame_selection
print "Genes inculded %s" % totalCountedTranscripts
print "Number of transcripts absent in UTRfile: %s" % noUTRentry
print "Number of transcripts with zero UTR lengths: %s" % zeroUtrlen
print "Number of transcripts with zero CDS density: %s" % zeroCdsdense
print "Number of transcripts below CDS density threshold: %s" % lowCdsdense
print "Number of transcripts with no In-frame stops: %s" % noInframeStops
print "Number of transcripts too close to normal stop codon: %s" % tooCloseToCDS
print "Number of transcripts with insufficient 3'UTR: %s" % utr3tooShort
print "- - - - - - - -"
print "RPM: %s" % averagegene
print "- - - - - - - -"
return averagegene