def init_basicVars(self, xOffset, sequence, ploidy, windowOverlap, readLen,
coverageDat):
self.x = xOffset
self.ploidy = ploidy
self.readLen = readLen
self.sequences = [bytearray(sequence) for n in xrange(self.ploidy)]
self.seqLen = len(sequence)
self.indelList = [[] for n in xrange(self.ploidy)]
self.snpList = [[] for n in xrange(self.ploidy)]
self.allCigar = [[] for n in xrange(self.ploidy)]
self.adj = [None for n in xrange(self.ploidy)]
# blackList[ploid][pos] = 0 safe to insert variant here
# blackList[ploid][pos] = 1 indel inserted here
# blackList[ploid][pos] = 2 snp inserted here
# blackList[ploid][pos] = 3 invalid position for various processing reasons
self.blackList = [
np.zeros(self.seqLen, dtype='<i4') for n in xrange(self.ploidy)
]
# disallow mutations to occur on window overlap points
self.winBuffer = windowOverlap
for p in xrange(self.ploidy):
self.blackList[p][-self.winBuffer] = 3
self.blackList[p][-self.winBuffer - 1] = 3
# if we're only creating a vcf, skip some expensive initialization related to coverage depth
if not self.onlyVCF:
(self.windowSize, coverage_vals) = coverageDat
self.win_per_read = int(self.readLen / float(self.windowSize) +
0.5)
self.which_bucket = DiscreteDistribution(coverage_vals,
range(len(coverage_vals)))
def init_trinucBias(self): # compute mutation positional bias given trinucleotide strings of the sequence (ONLY AFFECTS SNPs) # # note: since indels are added before snps, it's possible these positional biases aren't correctly utilized # at positions affected by indels. At the moment I'm going to consider this negligible. trinuc_snp_bias = [[0. for n in xrange(self.seqLen)] for m in xrange(self.ploidy)] self.trinuc_bias = [None for n in xrange(self.ploidy)] for p in xrange(self.ploidy): for i in xrange(self.winBuffer+1,self.seqLen-1): trinuc_snp_bias[p][i] = self.models[p][7][ALL_IND[str(self.sequences[p][i-1:i+2])]] self.trinuc_bias[p] = DiscreteDistribution(trinuc_snp_bias[p][self.winBuffer+1:self.seqLen-1],range(self.winBuffer+1,self.seqLen-1))
def init_mutModels(self, mutationModels, mutRate):
if mutationModels == []:
ml = [copy.deepcopy(DEFAULT_MODEL_1) for n in xrange(self.ploidy)]
self.modelData = ml[:self.ploidy]
else:
if len(mutationModels) != self.ploidy:
print '\nError: Number of mutation models recieved is not equal to specified ploidy\n'
exit(1)
self.modelData = copy.deepcopy(mutationModels)
# do we need to rescale mutation frequencies?
mutRateSum = sum([n[0] for n in self.modelData])
self.mutRescale = mutRate
if self.mutRescale == None:
self.mutScalar = 1.0
else:
self.mutScalar = float(
self.mutRescale) / (mutRateSum / float(len(self.modelData)))
# how are mutations spread to each ploid, based on their specified mut rates?
self.ploidMutFrac = [float(n[0]) / mutRateSum for n in self.modelData]
self.ploidMutPrior = DiscreteDistribution(self.ploidMutFrac,
range(self.ploidy))
# init mutation models
#
# self.models[ploid][0] = average mutation rate
# self.models[ploid][1] = p(mut is homozygous | mutation occurs)
# self.models[ploid][2] = p(mut is indel | mut occurs)
# self.models[ploid][3] = p(insertion | indel occurs)
# self.models[ploid][4] = distribution of insertion lengths
# self.models[ploid][5] = distribution of deletion lengths
# self.models[ploid][6] = distribution of trinucleotide SNP transitions
# self.models[ploid][7] = p(trinuc mutates)
self.models = []
for n in self.modelData:
self.models.append([
self.mutScalar * n[0], n[1], n[2], n[3],
DiscreteDistribution(n[5], n[4]),
DiscreteDistribution(n[7], n[6]), []
])
for m in n[8]:
self.models[-1][6].append([
DiscreteDistribution(m[0], NUCL),
DiscreteDistribution(m[1], NUCL),
DiscreteDistribution(m[2], NUCL),
DiscreteDistribution(m[3], NUCL)
])
self.models[-1].append([m for m in n[9]])
[GC_SCALE_COUNT, GC_SCALE_VAL] = pickle.load(open(GC_BIAS_MODEL,'rb')) GC_WINDOW_SIZE = GC_SCALE_COUNT[-1] # fragment length distribution # if PAIRED_END and not(PAIRED_END_ARTIFICIAL): print 'Using empirical fragment length distribution.' [potential_vals, potential_prob] = pickle.load(open(FRAGLEN_MODEL,'rb')) FRAGLEN_VALS = [] FRAGLEN_PROB = [] for i in xrange(len(potential_vals)): if potential_vals[i] > READLEN: FRAGLEN_VALS.append(potential_vals[i]) FRAGLEN_PROB.append(potential_prob[i]) # should probably add some validation and sanity-checking code here... FRAGLEN_DISTRIBUTION = DiscreteDistribution(FRAGLEN_PROB,FRAGLEN_VALS) FRAGMENT_SIZE = FRAGLEN_VALS[mean_ind_of_weighted_list(FRAGLEN_PROB)] # Indicate not writing FASTQ reads # if NO_FASTQ: print 'Bypassing FASTQ generation...' """************************************************ **** HARD-CODED CONSTANTS ************************************************""" # target window size for read sampling. how many times bigger than read/frag length WINDOW_TARGET_SCALE = 100 # sub-window size for read sampling windows. this is basically the finest resolution
def __init__(self, readLen, errorModel, reScaledError):
self.readLen = readLen
errorDat = pickle.load(open(errorModel,'rb'))
self.UNIFORM = False
if len(errorDat) == 4: # uniform-error SE reads (e.g. PacBio)
self.UNIFORM = True
[Qscores,offQ,avgError,errorParams] = errorDat
self.uniform_qscore = int(-10.*np.log10(avgError)+0.5)
print 'Using uniform sequencing error model. (q='+str(self.uniform_qscore)+'+'+str(offQ)+', p(err)={0:0.2f}%)'.format(100.*avgError)
if len(errorDat) == 6: # only 1 q-score model present, use same model for both strands
[initQ1,probQ1,Qscores,offQ,avgError,errorParams] = errorDat
self.PE_MODELS = False
elif len(errorDat) == 8: # found a q-score model for both forward and reverse strands
#print 'Using paired-read quality score profiles...'
[initQ1,probQ1,initQ2,probQ2,Qscores,offQ,avgError,errorParams] = errorDat
self.PE_MODELS = True
if len(initQ1) != len(initQ2) or len(probQ1) != len(probQ2):
print '\nError: R1 and R2 quality score models are of different length.\n'
exit(1)
self.qErrRate = [0.]*(max(Qscores)+1)
for q in Qscores:
self.qErrRate[q] = 10.**(-q/10.)
self.offQ = offQ
# errorParams = [SSE_PROB, SIE_RATE, SIE_PROB, SIE_VAL, SIE_INS_FREQ, SIE_INS_NUCL]
self.errP = errorParams
self.errSSE = [DiscreteDistribution(n,NUCL) for n in self.errP[0]]
self.errSIE = DiscreteDistribution(self.errP[2],self.errP[3])
self.errSIN = DiscreteDistribution(self.errP[5],NUCL)
# adjust sequencing error frequency to match desired rate
if reScaledError == None:
self.errorScale = 1.0
else:
self.errorScale = reScaledError/avgError
print 'Warning: Quality scores no longer exactly representative of error probability. Error model scaled by {0:.3f} to match desired rate...'.format(self.errorScale)
if self.UNIFORM == False:
# adjust length to match desired read length
if self.readLen == len(initQ1):
self.qIndRemap = range(self.readLen)
else:
print 'Warning: Read length of error model ('+str(len(initQ1))+') does not match -R value ('+str(self.readLen)+'), rescaling model...'
self.qIndRemap = [max([1,len(initQ1)*n/readLen]) for n in xrange(readLen)]
# initialize probability distributions
self.initDistByPos1 = [DiscreteDistribution(initQ1[i],Qscores) for i in xrange(len(initQ1))]
self.probDistByPosByPrevQ1 = [None]
for i in xrange(1,len(initQ1)):
self.probDistByPosByPrevQ1.append([])
for j in xrange(len(initQ1[0])):
if np.sum(probQ1[i][j]) <= 0.: # if we don't have sufficient data for a transition, use the previous qscore
self.probDistByPosByPrevQ1[-1].append(DiscreteDistribution([1],[Qscores[j]],degenerateVal=Qscores[j]))
else:
self.probDistByPosByPrevQ1[-1].append(DiscreteDistribution(probQ1[i][j],Qscores))
if self.PE_MODELS:
self.initDistByPos2 = [DiscreteDistribution(initQ2[i],Qscores) for i in xrange(len(initQ2))]
self.probDistByPosByPrevQ2 = [None]
for i in xrange(1,len(initQ2)):
self.probDistByPosByPrevQ2.append([])
for j in xrange(len(initQ2[0])):
if np.sum(probQ2[i][j]) <= 0.: # if we don't have sufficient data for a transition, use the previous qscore
self.probDistByPosByPrevQ2[-1].append(DiscreteDistribution([1],[Qscores[j]],degenerateVal=Qscores[j]))
else:
self.probDistByPosByPrevQ2[-1].append(DiscreteDistribution(probQ2[i][j],Qscores))
def init_coverage(self,coverageDat,fragDist=None):
# if we're only creating a vcf, skip some expensive initialization related to coverage depth
if not self.onlyVCF:
(self.windowSize, gc_scalars, targetCov_vals) = coverageDat
gcCov_vals = [[] for n in self.sequences]
trCov_vals = [[] for n in self.sequences]
self.coverage_distribution = []
avg_out = []
for i in xrange(len(self.sequences)):
# compute gc-bias
j = 0
while j+self.windowSize < len(self.sequences[i]):
gc_c = self.sequences[i][j:j+self.windowSize].count('G') + self.sequences[i][j:j+self.windowSize].count('C')
gcCov_vals[i].extend([gc_scalars[gc_c]]*self.windowSize)
j += self.windowSize
gc_c = self.sequences[i][-self.windowSize:].count('G') + self.sequences[i][-self.windowSize:].count('C')
gcCov_vals[i].extend([gc_scalars[gc_c]]*(len(self.sequences[i])-len(gcCov_vals[i])))
#
trCov_vals[i].append(targetCov_vals[0])
prevVal = self.FM_pos[i][0]
for j in xrange(1,len(self.sequences[i])-self.readLen):
if self.FM_pos[i][j] == None:
trCov_vals[i].append(targetCov_vals[prevVal])
else:
trCov_vals[i].append(sum(targetCov_vals[self.FM_pos[i][j]:self.FM_span[i][j]])/float(self.FM_span[i][j]-self.FM_pos[i][j]))
prevVal = self.FM_pos[i][j]
#print (i,j), self.adj[i][j], self.allCigar[i][j], self.FM_pos[i][j], self.FM_span[i][j]
# shift by half of read length
trCov_vals[i] = [0.0]*int(self.readLen/2) + trCov_vals[i][:-int(self.readLen/2.)]
# fill in missing indices
trCov_vals[i].extend([0.0]*(len(self.sequences[i])-len(trCov_vals[i])))
#
covvec = np.cumsum([trCov_vals[i][nnn]*gcCov_vals[i][nnn] for nnn in xrange(len(trCov_vals[i]))])
coverage_vals = []
for j in xrange(0,len(self.sequences[i])-self.readLen):
coverage_vals.append(covvec[j+self.readLen] - covvec[j])
avg_out.append(np.mean(coverage_vals)/float(self.readLen))
if fragDist == None:
self.coverage_distribution.append(DiscreteDistribution(coverage_vals,range(len(coverage_vals))))
# fragment length nightmare
else:
currentThresh = 0.
index_list = [0]
for j in xrange(len(fragDist.cumP)):
if fragDist.cumP[j] >= currentThresh + COV_FRAGLEN_PERCENTILE/100.0:
currentThresh = fragDist.cumP[j]
index_list.append(j)
flq = [fragDist.values[nnn] for nnn in index_list]
if fragDist.values[-1] not in flq:
flq.append(fragDist.values[-1])
flq.append(LARGE_NUMBER)
self.fraglens_indMap = {}
for j in fragDist.values:
bInd = bisect.bisect(flq,j)
if abs(flq[bInd-1] - j) <= abs(flq[bInd] - j):
self.fraglens_indMap[j] = flq[bInd-1]
else:
self.fraglens_indMap[j] = flq[bInd]
self.coverage_distribution.append({})
for flv in sorted(list(set(self.fraglens_indMap.values()))):
buffer_val = self.readLen
for j in fragDist.values:
if self.fraglens_indMap[j] == flv and j > buffer_val:
buffer_val = j
coverage_vals = []
for j in xrange(len(self.sequences[i])-buffer_val):
coverage_vals.append(covvec[j+self.readLen] - covvec[j] + covvec[j+flv] - covvec[j+flv-self.readLen])
# EXPERIMENTAL
#quantized_covVals = quantize_list(coverage_vals)
#self.coverage_distribution[i][flv] = DiscreteDistribution([n[2] for n in quantized_covVals],[(n[0],n[1]) for n in quantized_covVals])
# TESTING
#import matplotlib.pyplot as mpl
#print len(coverage_vals),'-->',len(quantized_covVals)
#mpl.figure(0)
#mpl.plot(range(len(coverage_vals)),coverage_vals)
#for qcv in quantized_covVals:
# mpl.plot([qcv[0],qcv[1]+1],[qcv[2],qcv[2]],'r')
#mpl.show()
#exit(1)
self.coverage_distribution[i][flv] = DiscreteDistribution(coverage_vals,range(len(coverage_vals)))
return np.mean(avg_out)
def parseFQ(inf):
print 'reading ' + inf + '...'
if inf[-3:] == '.gz':
print 'detected gzip suffix...'
f = gzip.open(inf, 'r')
else:
f = open(inf, 'r')
IS_SAM = False
if inf[-4:] == '.sam':
print 'detected sam input...'
IS_SAM = True
rRead = 0
actual_readlen = 0
qDict = {}
while True:
if IS_SAM:
data4 = f.readline()
if not len(data4):
break
try:
data4 = data4.split('\t')[10]
except IndexError:
break
# need to add some input checking here? Yup, probably.
else:
data1 = f.readline()
data2 = f.readline()
data3 = f.readline()
data4 = f.readline()
if not all([data1, data2, data3, data4]):
break
if actual_readlen == 0:
if inf[-3:] != '.gz' and not IS_SAM:
totalSize = os.path.getsize(inf)
entrySize = sum([len(n) for n in [data1, data2, data3, data4]])
print 'estimated number of reads in file:', int(
float(totalSize) / entrySize)
actual_readlen = len(data4) - 1
print 'assuming read length is uniform...'
print 'detected read length (from first read found):', actual_readlen
priorQ = np.zeros([actual_readlen, RQ])
totalQ = [None] + [
np.zeros([RQ, RQ]) for n in xrange(actual_readlen - 1)
]
# sanity-check readlengths
if len(data4) - 1 != actual_readlen:
print 'skipping read with unexpected length...'
continue
for i in range(len(data4) - 1):
q = ord(data4[i]) - offQ
qDict[q] = True
if i == 0:
priorQ[i][q] += 1
else:
totalQ[i][prevQ, q] += 1
priorQ[i][q] += 1
prevQ = q
rRead += 1
if rRead % PRINT_EVERY == 0:
print rRead
if MAX_READS > 0 and rRead >= MAX_READS:
break
f.close()
# some sanity checking again...
QRANGE = [min(qDict.keys()), max(qDict.keys())]
if QRANGE[0] < 0:
print '\nError: Read in Q-scores below 0\n'
exit(1)
if QRANGE[1] > RQ:
print '\nError: Read in Q-scores above specified maximum:', QRANGE[
1], '>', RQ, '\n'
exit(1)
print 'computing probabilities...'
probQ = [None] + [[[0. for m in xrange(RQ)] for n in xrange(RQ)]
for p in xrange(actual_readlen - 1)]
for p in xrange(1, actual_readlen):
for i in xrange(RQ):
rowSum = float(np.sum(totalQ[p][i, :])) + PROB_SMOOTH * RQ
if rowSum <= 0.:
continue
for j in xrange(RQ):
probQ[p][i][j] = (totalQ[p][i][j] + PROB_SMOOTH) / rowSum
initQ = [[0. for m in xrange(RQ)] for n in xrange(actual_readlen)]
for i in xrange(actual_readlen):
rowSum = float(np.sum(priorQ[i, :])) + INIT_SMOOTH * RQ
if rowSum <= 0.:
continue
for j in xrange(RQ):
initQ[i][j] = (priorQ[i][j] + INIT_SMOOTH) / rowSum
if PLOT_STUFF:
mpl.rcParams.update({
'font.size': 14,
'font.weight': 'bold',
'lines.linewidth': 3
})
mpl.figure(1)
Z = np.array(initQ).T
X, Y = np.meshgrid(range(0, len(Z[0]) + 1), range(0, len(Z) + 1))
mpl.pcolormesh(X, Y, Z, vmin=0., vmax=0.25)
mpl.axis([0, len(Z[0]), 0, len(Z)])
mpl.yticks(range(0, len(Z), 10), range(0, len(Z), 10))
mpl.xticks(range(0, len(Z[0]), 10), range(0, len(Z[0]), 10))
mpl.xlabel('Read Position')
mpl.ylabel('Quality Score')
mpl.title('Q-Score Prior Probabilities')
mpl.colorbar()
mpl.show()
VMIN_LOG = [-4, 0]
minVal = 10**VMIN_LOG[0]
qLabels = [
str(n) for n in range(QRANGE[0], QRANGE[1] + 1) if n % 5 == 0
]
print qLabels
qTicksx = [int(n) + 0.5 for n in qLabels]
qTicksy = [(RQ - int(n)) - 0.5 for n in qLabels]
for p in xrange(1, actual_readlen, 10):
currentDat = np.array(probQ[p])
for i in xrange(len(currentDat)):
for j in xrange(len(currentDat[i])):
currentDat[i][j] = max(minVal, currentDat[i][j])
# matrix indices: pcolormesh plotting: plot labels and axes:
#
# y ^ ^
# --> x | y |
# x | --> -->
# v y x
#
# to plot a MxN matrix 'Z' with rowNames and colNames we need to:
#
# pcolormesh(X,Y,Z[::-1,:]) # invert x-axis
# # swap x/y axis parameters and labels, remember x is still inverted:
# xlim([yMin,yMax])
# ylim([M-xMax,M-xMin])
# xticks()
#
mpl.figure(p + 1)
Z = np.log10(currentDat)
X, Y = np.meshgrid(range(0, len(Z[0]) + 1), range(0, len(Z) + 1))
mpl.pcolormesh(X,
Y,
Z[::-1, :],
vmin=VMIN_LOG[0],
vmax=VMIN_LOG[1],
cmap='jet')
mpl.xlim([QRANGE[0], QRANGE[1] + 1])
mpl.ylim([RQ - QRANGE[1] - 1, RQ - QRANGE[0]])
mpl.yticks(qTicksy, qLabels)
mpl.xticks(qTicksx, qLabels)
mpl.xlabel('\n' + r'$Q_{i+1}$')
mpl.ylabel(r'$Q_i$')
mpl.title('Q-Score Transition Frequencies [Read Pos:' + str(p) +
']')
cb = mpl.colorbar()
cb.set_ticks([-4, -3, -2, -1, 0])
cb.set_ticklabels([
r'$10^{-4}$', r'$10^{-3}$', r'$10^{-2}$', r'$10^{-1}$',
r'$10^{0}$'
])
#mpl.tight_layout()
mpl.show()
print 'estimating average error rate via simulation...'
Qscores = range(RQ)
#print (len(initQ), len(initQ[0]))
#print (len(probQ), len(probQ[1]), len(probQ[1][0]))
initDistByPos = [
DiscreteDistribution(initQ[i], Qscores) for i in xrange(len(initQ))
]
probDistByPosByPrevQ = [None]
for i in xrange(1, len(initQ)):
probDistByPosByPrevQ.append([])
for j in xrange(len(initQ[0])):
if np.sum(
probQ[i][j]
) <= 0.: # if we don't have sufficient data for a transition, use the previous qscore
probDistByPosByPrevQ[-1].append(
DiscreteDistribution([1], [Qscores[j]],
degenerateVal=Qscores[j]))
else:
probDistByPosByPrevQ[-1].append(
DiscreteDistribution(probQ[i][j], Qscores))
countDict = {}
for q in Qscores:
countDict[q] = 0
for samp in xrange(1, N_SAMP + 1):
if samp % PRINT_EVERY == 0:
print samp
myQ = initDistByPos[0].sample()
countDict[myQ] += 1
for i in xrange(1, len(initQ)):
myQ = probDistByPosByPrevQ[i][myQ].sample()
countDict[myQ] += 1
totBases = float(sum(countDict.values()))
avgError = 0.
for k in sorted(countDict.keys()):
eVal = 10.**(-k / 10.)
#print k, eVal, countDict[k]
avgError += eVal * (countDict[k] / totBases)
print 'AVG ERROR RATE:', avgError
return (initQ, probQ, avgError)
if print_path:
print('Path cost is', discovered[target][1])
stack = []
curr = target
while curr:
stack.append((curr, self.original_universe[curr[0]][curr[1]]))
print(curr)
curr = discovered[curr][0]
print('Path from start to target:', stack[::-1])
return discovered[target][1]
if __name__ == '__main__':
state = load_maze()
universe = state.universe
start, target = state.start, state.target
portals = state.portals
discrete_distribution = DiscreteDistribution(portals)
heuristics = Heuristic(portals, target, discrete_distribution)
solver = A_Star(universe, portals, start, target, discrete_distribution)
report = make_statistics(solver, heuristics, discrete_distribution)
print(report)
