def gatherCausalStats(Ddata,
scenario,
selpos=500000,
thinSfx='',
complikeSfx=None,
likesTableSfx='',
nonNanStats='ALL',
getio=None):
"""For each replica in one scenario, gather causal SNP stats and save them as the replica statistic.
"""
snpStatsDir = os.path.join(Ddata, 'snpStats' + thinSfx, scenario.scenDir())
replicaStatsDir = os.path.join(Ddata, 'replicastats' + thinSfx,
scenario.scenDir())
complikeFN = os.path.join(
snpStatsDir,
AddFileSfx('complike.data/', 'normedLocal', scenario.mutPop,
complikeSfx, 'nonNan', *MakeSeq(nonNanStats)))
causalStatsFN = os.path.join(
replicaStatsDir,
AddFileSfx('causalStats.tsv', complikeSfx, likesTableSfx, 'nonNan',
*MakeSeq(nonNanStats)))
if getio:
return dict(depends_on=complikeFN,
creates=causalStatsFN,
mediumRuleNameSfx=(scenario.scenDir(), complikeSfx,
likesTableSfx))
complikeFile = IDotData(complikeFN)
complikeFile[complikeFile.Pos == selpos].addComputedCols(
newColNames='replicaNum',
newColFn=lambda r: int(r.Chrom)).save(causalStatsFN)
def gatherCausalRanks(Ddata=None,
scenario=None,
selpos=500000,
thinSfx='',
complikeSfx=None,
likesTableSfx='',
nonNanStats='ALL',
cmsFileFN=None,
causalRankFN=None,
getio=None):
"""For each replica in one scenario, get the rank of the causal SNP by CMS score, and save as a replica statistic.
"""
assert cmsFileFN or (Ddata and scenario)
scenDir = scenario.scenDir() if scenario else 'unknown_scenDir'
if not cmsFileFN:
snpStatsDir = os.path.join(Ddata, 'snpStats' + thinSfx, scenDir)
if not cmsFileFN:
cmsFileFN = os.path.join(
snpStatsDir,
AddFileSfx('complike.data/', scenario.mutPop, complikeSfx,
likesTableSfx))
if not causalRankFN:
causalRankFN = os.path.join(
Ddata, 'replicastats', scenDir,
AddFileSfx('causalRank.tsv', complikeSfx, 'nonNan',
*MakeSeq(nonNanStats)))
if getio:
return dict(depends_on=cmsFileFN,
creates=causalRankFN,
mediumRuleNameSfx=(scenDir, complikeSfx))
cmsScores = IDotData(cmsFileFN)
if nonNanStats.upper() == 'ALL': nonNanStats = cmsScores.headings
with IDotData.openForWrite(
causalRankFN,
headings='replicaNum causalRank causalScore') as causalRankFile:
for replicaNum, cmsScores1, cmsScores2 in cmsScores.groupby(
'Chrom', multiPass=2):
for r1 in cmsScores1:
if r1.Pos == selpos:
causalScore = r1.complike
numHigher = 0
for r2 in cmsScores2:
if r2.complike > causalScore and all(
[np.isfinite(r2[c]) for c in nonNanStats]):
numHigher += 1
causalRankFile.writeRecord(int(replicaNum), numHigher,
causalScore)
if r1.Pos >= selpos: break
def plotHistograms(inFN, cols, outFNs=None, getio=None, **kwargs):
histFNs = GetCreates(computeHistograms, **Dict('inFN cols outFNs'))
outFN = AddFileSfx(ReplaceFileExt(inFN, '.svg'), 'hist')
if getio: return dict(depends_on=histFNs, creates=outFN)
GraphHistograms(histFiles=histFNs,
outFile=outFN,
labels=tuple(MakeSeq(cols)),
**kwargs)
def mergePosFilesOneSim( scenario, Ddata, replicaNum, putativeMutPop = None, simsOut = 'simsOut', thinExt = '', thinSfx = '',
statsSfx = '', ihsSfx = '', pop2name = pop2name,
getio = None ):
"""Merge per-SNP position files for one sim.
"""
if not Ddata.endswith('/'): Ddata += '/'
popNums = sorted( pop2name.keys() )
minPopNum = popNums[ 0 ]
SimDir = os.path.join( Ddata, simsOut + thinSfx )
scenName = scenario.scenName()
scenDir = scenario.scenDir()
if putativeMutPop == None: putativeMutPop = scenario.mutPop
posFiles = [ os.path.join( SimDir, scenDir, '%d_%s.pos-%d%s' % ( replicaNum, scenName, pop, thinExt ) )
for pop in popNums ]
snpStatsDir = os.path.join( Ddata, 'snpStats' + thinSfx, scenario.scenDir() )
mergedPosFN = os.path.join( snpStatsDir, 'mergedPos', AddFileSfx( 'mergedPos.tsv', statsSfx, putativeMutPop, ihsSfx, replicaNum ) )
if getio: return dict( depends_on = posFiles, creates = mergedPosFN,
splitByCols = dict([ ( posFile, dict( keyCols = 'CHROM_POS',
tableReadOpts = dict( headingSep = 'whitespace' ) ) ) for posFile in posFiles ]),
mediumRuleNameSfx = scenario.scenDir() )
replicaIDotDatas = []
posCols = ['CHROM_POS %d' % minPopNum ] + ['FREQ1 %d' % popNum for popNum in popNums]
# Load in pos files for this replica.
# They give, for each SNP in the replica, its physical (basepair) position within the replica,
# and the frequency of the derived and the ancestral alleles.
posIndiv = [ IDotData( os.path.join( SimDir, scenDir, str(replicaNum) + '_' + scenName +
'.pos-%d%s' % ( pop, thinExt) ), skipFirstLines = 1,
headings = ('SNP','CHROM', 'CHROM_POS', 'ALLELE1',
'FREQ1', 'ALLELE2', 'FREQ2' ))
for pop in popNums ]
posAll = IDotData.merge( iDotDatas = posIndiv,
cols = ('CHROM_POS',) * len( posIndiv ),
blanks = (np.nan,) * len( posIndiv ),
suffixes = [ ' %s' % pop for pop in popNums ] )
posAll = posAll[posCols]
IDotData.repeat( heading = 'replicaNum', value = replicaNum ).hstack( posAll ).save( mergedPosFN )
def recordCount(inFN, outFN=None, getio=None):
"""Count the lines in the given file"""
if outFN is None: outFN = AddFileSfx(inFN, 'recordCount')
def SaveCount(count, outFN):
IDotData(names=('recordCount', ), Records=(count, )).save(outFN)
def combiner(inFNs, outFN):
SaveCount(np.sum([next(iter(IDotData(f))) for f in inFNs]), outFN)
if getio:
return dict(depends_on=inFN,
creates=outFN,
splitByCols={inFN: dict(keyCols=())},
combiner={outFN: combiner})
contents = SlurpFile(inFN).strip()
SaveCount(contents.count('\n'), outFN)
def computeMeanStdWithinGroups(inFN,
cols,
groupCols,
groupsAreContiguous=True,
outFN=None,
getio=None):
"""Add columns representing mean and std within each group.
"""
sumsFN = GetCreates(computeSumsWithinGroups,
**Dict('inFN cols groupCols groupsAreContiguous'))[0]
if outFN is None:
outFN = AddFileSubdir('stats',
AddFileSfx(inFN, 'meanStd', *(cols + groupCols)))
if getio:
return dict(depends_on=sumsFN,
creates=outFN,
attrs=dict(piperun_short=True))
return IDotData(sumsFN).addMeanStdCols(cols=cols).save(outFN)
def computeHistograms(inFN, cols, binSizes=None, outFNs=None, getio=None):
"""Compute histograms of the specified columns of the input"""
cols = tuple(MakeSeq(cols))
binSizesHere = (.001, ) * len(cols) if binSizes is None else tuple(
MakeSeq(binSizes))
outFNsHere = outFNs
if outFNsHere is None:
outFNsHere = [
AddFileSubdir('stats', AddFileSfx(inFN, 'hist', col))
for col in cols
]
assert len(cols) == len(binSizesHere) == len(outFNsHere)
if getio: return dict(depends_on=inFN, creates=outFNsHere)
# add histogram combiner
hists = [Histogrammer(binSize=binSize) for binSize in binSizesHere]
z = IDotData(inFN)
for h, c, outFN in zip(hists, cols, outFNsHere):
h.addVals(z[c])
h.save(outFN)
def computeSumsWithinGroups(inFN,
cols,
groupCols,
groupsAreContiguous=True,
outFN=None,
getio=None):
"""For a tsv file, compute sums, sumsquares and counts for each of the given columns within groups
defined by groupCols.
>>> z = IDotData( names = ( 'a', 'b' ), Records = ( ( 1, 2 ), ( 1, 3 ), ( 2, 4 ), ( 2, 5 ) ) )
>>> computeSumsWithinGroups( inFN = z, cols = 'b', groupCols = 'a', outFN = sys.stdout )
... # doctest: +NORMALIZE_WHITESPACE
a b_count b_sum b_sumSq b_numNaN
1 2 5.0 13.0 0
2 2 9.0 41.0 0
"""
cols = tuple(MakeSeq(cols))
groupCols = tuple(MakeSeq(groupCols))
if outFN is None:
outFN = AddFileSubdir('stats',
AddFileSfx(inFN, 'sums', *(cols + groupCols)))
def combiner(inFNs, outFN):
IDotData.mergeColumnSummaries(iDotDatas=inFNs,
cols=cols,
groupCols=groupCols).save(outFN)
if getio:
return dict(depends_on=inFN,
creates=outFN,
splitByCols={inFN: dict(keyCols=())},
combiner={outFN: combiner})
IDotData(inFN).summarizeColumnsWithinGroups(
**Dict('cols groupCols groupsAreContiguous')).save(outFN)
def mergeSims(scenario,
Ddata='../Data/Shari_Data/sim/',
simsOut='simsOut3',
nreplicas=5,
thinExt='.thin',
thinSfx='',
selpop=None,
getio=None):
"""Gathers per-SNP information, for all replicas of a given scenario, and outputs it in a single DotData where each line
gives info for one SNP.
Specifically, reads simulation and Sweep output, collects columns needed for composite likehood test (chrom, base pair position, genetic
distance, anc frequencies for 3 populations, xpop for each pair, and ihs, iHH_A and iHH_D for selected population)
Input params:
scenario - an object of class Scenario, indicating the simulation scenario (either neutral or a selection scenario)
from which all replicas were simulated.
nreplicas - the number of replicas simulated under this scenario.
Each replica represents a chromosome region, with a set of SNPs on it.
Ddata - the directory under which the simulations and the Sweep analysis results live.
Under this directory we expect to find:
iHS analysis results, under power_ihs/
XP-EHH analysis results, under power_xpop
simulation output giving SNP positions
thinExt - the extension appended to simulation files that describe the SNPs in the simulated replica.
Sometimes we create simulations and then thin them under different thinning models (to simulate SNP ascertainment
by the various stages of HapMap; these differently thinned versions of the same simulations might be stored in
simulation files with different extensions.
thinSfx - the suffix appended to the power_ihs and power_xpop directory names, telling where to find iHS and XP-EHH
analyses of the simulations. When we analyze the same simulations after applying different thinning scenarios,
the iHS and XP-EHH analyses for each thinning scenario go into a separate set of directories.
Output params:
Ddata - under Ddata writes a DotData named merged_scenName.data, where each line gives info
for one SNP, with the following columns (type of data is float unless stated otherwise):
CHROM_POS 1 - physical (basepair) position of the SNP within its replica.
Note that one merged file contains SNPs for a set of replicas (all for the same scenario),
so there could be multiple SNPs with the same position. The replica number
is given in the Chrom column.
FREQ1 1 - derived allele frequency in pop 1 ( European )
FREQ1 4 - derived allele frequency in pop 4 ( EastAsian )
FREQ1 5 - derived allele frequency in pop 5 ( WestAfrican )
R AllEHH logratio Deviation European_WestAfrican - XP-EHH score to the right of the SNP,
between European and WestAfrican pops, normalized to the neutral background.
Analogously for the next five columns:
L AllEHH logratio Deviation European_WestAfrican
R AllEHH logratio Deviation EastAsian_European
L AllEHH logratio Deviation EastAsian_European
R AllEHH logratio Deviation EastAsian_WestAfrican
L AllEHH logratio Deviation EastAsian_WestAfrican
SNP pos (cM) European_WestAfrican - genetic map position of this SNP, within its replica.
(the European_WestAfrican suffix is irrelevant).
SNP pos (bases) European_WestAfrican - physical (basepair) position of this SNP within its replica.
(the European_WestAfrican suffix is irrelevant).
Chrom European_WestAfrican - the replica from which this SNP comes; can be nan.
(the European_WestAfrican suffix is irrelevant)
Chrom - the replica from which this SNP comes; can be nan
SNP pos (bases) - physical (basepair) position of this SNP within its replica.
SNP pos (cM) - genetic map position of this SNP within its replica
Both iHH_A - sum of iHH_A for both directions from this SNP
Both iHH_D - sum of iHH_D for both directions from this SNP
Both iHS - the value in 'Both Unstandardised iHS' (below), but binned by derived allele frequency
and normalized within the bin.
Left iHH_D - iHH_D to the left of the SNP (the raw integral value). analogously for the next three.
Right iHH_D
Left iHH_A
Right iHH_A
Both Unstandardised iHS - log( (iHH_A_left + iHH_A_right) / ( iHH_D_left + iHH_D_right ) )
( see also 'Both iHS' column for the standardized iHS score )
"""
assert selpop == None or scenario.is_neutral()
DataDir = Ddata + '/'
SimDir = DataDir + simsOut + thinSfx + '/'
if not scenario.is_neutral():
scenName = 'sel%d_%d' % (scenario.mutFreq, scenario.mutPop)
scenDir = str(scenario.mutAge) + 'ky/' + scenName
else:
scenName = 'neutral'
scenDir = 'neutral'
popName = {1: 'European', 4: 'EastAsian', 5: 'WestAfrican'}
ihsSignifTsv = DataDir + 'power_ihs' + thinSfx + '/' + scenDir + '/ihs_sig_' + \
popName[ scenario.mutPop if not scenario.is_neutral() else ( selpop if selpop != None else 1 ) ] + '.tsv'
xpopSignifTsv = [
DataDir + 'power_xpop' + thinSfx + '/' + scenDir +
'/xpop_significance_' + popPair + '.tsv'
for popPair in ('EastAsian_WestAfrican', 'EastAsian_European',
'European_WestAfrican')
]
posFiles = [
SimDir + scenDir + '/' + str(ichrom) + '_' + scenName + '.pos-%d%s' %
(pop, thinExt) for ichrom in range(nreplicas) for pop in (1, 4, 5)
]
ageSfx = '%dky' % (scenario.mutAge if not scenario.isNeutral() else 10)
mergedDotData = AddFileSfx(Ddata + 'merged.data/', ageSfx,
scenario.scenName(), selpop, thinSfx)
fileDescrs = \
{ mergedDotData :
( 'Various per-snp statistics for SNPs in scenario $scenario, replicas 0-$nreplicas.',
( ( 'CHROM_POS 1', 'physical (basepair) position of the SNP within its replica. '
'Note that one merged file contains SNPs for a set of replicas (all for the same scenario), '
'so there could be multiple SNPs with the same position. The replica number '
'is given in the Chrom column. ' ),
( 'FREQ1 1', 'derived allele frequency in pop 1 ( European )' ),
( 'R AllEHH logratio Deviation European_WestAfrican', 'XP-EHH score to the R of the SNP, '
'between European and WestAfrican pops, normalized to the neutral background.' ),
( 'SNP pos (cM) European_WestAfrican', 'genetic map SNP position' ),
( 'SNP pos (bases) European_WestAfrican', 'physical SNP position' ),
( 'Chrom European_WestAfrican', 'chromosome (or replica number)' ),
( 'Chrom', 'chromosome (or replica number)' ),
( 'SNP pos (bases)', 'physical SNP position' ),
( 'SNP pos (cM)', 'genetic map SNP position' ),
( 'Both iHH_A', 'sum of iHH_A scores for both sides' ),
( 'Both iHH_D', 'sum of iHH_D scores for both sides' ),
( 'Both iHS', 'sum of iHS scores for both sides' ),
( ' Left iHH_D', 'iHH_D score to the left of the SNP' ),
( 'Right iHH_D', 'iHH_D score to the right of the SNP' ),
( 'Left iHH_A', 'iHH_A score to the left of the SNP' ),
( 'Right iHH_A', 'iHH_A score to the right of the SNP' ),
( 'Both Unstandardised iHS', 'sum of unstandardized iHS scores for both sides' ) ) ) }
if getio:
return dict(depends_on=posFiles + [ihsSignifTsv] + xpopSignifTsv,
creates=mergedDotData,
mediumRuleNameSfx=scenario.scenDir(),
fileDescrs=fileDescrs)
ncausal = 0
dashFixer = lambda v: v if v != '-' else numpy.nan
# Load iHS of selected pop
ihsAll = DotData(SVPath=ihsSignifTsv,
ToLoad=[
'Chrom', 'SNP pos (bases)', 'SNP pos (cM)',
'Both iHH_A', 'Both iHH_D', 'Both iHS', 'Left iHH_D',
'Right iHH_D', 'Left iHH_A', 'Right iHH_A',
'Both Unstandardised iHS'
],
SVValueFixer=dashFixer)
ihsAllChrom = ihsAll.Chrom
# Load xpop values
xpopAll = xpopMerge(*xpopSignifTsv)
logging.info('done with xpopMerge')
xpopAll = xpopAll[[
'R AllEHH logratio Deviation European_WestAfrican',
'L AllEHH logratio Deviation European_WestAfrican',
'R AllEHH logratio Deviation EastAsian_European',
'L AllEHH logratio Deviation EastAsian_European',
'R AllEHH logratio Deviation EastAsian_WestAfrican',
'L AllEHH logratio Deviation EastAsian_WestAfrican',
'SNP pos (cM) European_WestAfrican',
'SNP pos (bases) European_WestAfrican', 'Chrom European_WestAfrican'
]]
xpopAllChrom = xpopAll['Chrom European_WestAfrican']
replicates = []
xpopIdx = 0
ihsIdx = 0
for ichrom in range(nreplicas):
progress('Merging replicas', ichrom, nreplicas, freq=1)
logging.info('looking at replica %d of %d' % (ichrom, nreplicas))
# Load in pos files for this replica.
# They give, for each SNP in the replica, its physical (basepair) position within the replica,
# and the frequency of the derived and the ancestral alleles.
pos1, pos4, pos5 = [
DotData(SVPath=SimDir + scenDir + '/' + str(ichrom) + '_' +
scenName + '.pos-%d%s' % (pop, thinExt),
SVSkipFirstLines=1,
SVHeader=False,
names=[
'SNP', 'CHROM', 'CHROM_POS', 'ALLELE1', 'FREQ1',
'ALLELE2', 'FREQ2'
]) for pop in (1, 4, 5)
]
assert pos1.numCols() == pos4.numCols() == pos5.numCols()
posBlank = ((numpy.nan, ) * pos1.numCols(), ) * 3
logging.info('Loaded pos files for chrom ' + str(ichrom) + ': ' +
str(len(pos1)) + 'snps')
assert set(pos1.CHROM_POS) == set(pos4.CHROM_POS) == set(
pos5.CHROM_POS)
logging.info('pos file sizes are: %d, %d, %d' %
(len(pos1), len(pos4), len(pos5)))
logging.info('Merging on position...')
posAll = DotData.mergeOnKeyCols((pos1, pos4, pos5),
('CHROM_POS', ) * 3,
posBlank,
suffixes=(' 1', ' 4', ' 5'))
logging.info('Done merging.')
logging.info('type(posAll) is ' + str(type(posAll)))
print len(posAll)
chrom = numpy.ones(len(posAll)) * ichrom
newChrom = DotData(Columns=[
chrom,
], names=[
'newChrom',
])
print newChrom
posAll = posAll[['CHROM_POS 1', 'FREQ1 1', 'FREQ1 4', 'FREQ1 5']]
posAll.hstack(newChrom)
logging.info('added replica number column')
print posAll
posAllBlank = (numpy.nan, ) * posAll.numCols()
# 10-16-08 ADDED CHROM TO MERGED OUTPT ( not now used -- can be removed? )
#
# From the xpop and ihs significance results, get just the rows for SNPs in the
# current replica
#
#while xpopIdx < len( xpopAllChrom ) and xpopAllChrom[ xpopIdx ] == ichrom: xpopIdx += 1
#xpop = xpopAll[ :xpopIdx ]
xpop = xpopAll[xpopAllChrom == ichrom]
logging.info('selected xpop for replica %d' % ichrom)
xpopBlank = (numpy.nan, ) * xpop.numCols()
#while ihsIdx < len( ihsAllChrom ) and ihsAllChrom[ ihsIdx ] == ichrom: ihsIdx += 1
#ihs = ihsAll[ :ihsIdx ]
ihs = ihsAll[ihsAllChrom == ichrom]
logging.info('selected ihs for replica %d' % ichrom)
ihsBlank = (numpy.nan, ) * ihs.numCols()
# if not set( ihs[ 'SNP pos (bases)' ] ).issubset( set( posAll['CHROM_POS 1'] ) ):
# print 'bad positions: ', set( posAll['CHROM_POS 1'] ) - set( ihs[ 'SNP pos (bases)' ] )
# assert set( ihs[ 'SNP pos (bases)' ] ).issubset( set( posAll['CHROM_POS 1'] ) ), "bad iHS file " + ihsSignifTsv
logging.info('merging replica %d' % ichrom)
Data = DotData.mergeOnKeyCols(
(posAll, xpop, ihs),
('CHROM_POS 1', 'SNP pos (bases) European_WestAfrican',
'SNP pos (bases)'),
blanks=(posAllBlank, xpopBlank, ihsBlank),
suffixes=('pos', ' xpop', ' ihs'),
verbose=True)
logging.info('done merging replica %d; now have %d records' %
(ichrom, len(Data)))
Data = Data[numpy.invert(numpy.isnan(Data['CHROM_POS 1']))]
logging.info(
'done removing snp info for SNPs not in all .pos files for replica %d; now have %d records'
% (ichrom, len(Data)))
replicates.append(Data)
logging.info('now have ' + str(len(replicates)) + ' replicates.')
# endloop: for each replica
logging.info('Stacking replicates...')
allData = reduce(lambda x, y: x.vstack(y), replicates)
logging.info('Saving merged SNP info to ' + mergedDotData)
allData.save(mergedDotData)
logging.info('Finished mergeSims()')
def DefineRulesTo_fastCMS(pr,
pops,
chroms,
selPop,
sweepDir,
cmsDir,
genomeBuild='hg19'):
"""Define rules to do fast CMS computation.
Params:
pr - the PipeRun object to which to add rules
selPop - testing selection in which pop?
pops - comparing selPop to which pops?
sweepDir - the sweep directory
cmsDir - the directory under which CMS stats go
"""
pops = list(MakeSeq(pops))
if selPop not in pops: pops.append(selPop)
allPops = tuple(MakeSeq(pops))
if selPop not in allPops: allPops += (selPop, )
cmpPops = [pop for pop in allPops if pop != selPop]
rawScoresFN = {}
genMapSfx = genomeBuild2genMapSfx[genomeBuild]
for pop in allPops:
for chrom in chroms:
with pr.settingAttrs('pop chrom'):
snpInfoFN = os.path.join(
sweepDir,
'analysis/chr%(chrom)s/snps_%(pop)s.tsv' % locals())
projDir = os.path.join(sweepDir,
'data/chr%(chrom)s' % locals())
ancestralImportedFN = os.path.join(projDir,
'ancestral.tsv.imported')
genotypesImportedFN = os.path.join(
projDir,
'genotypes_chr%(chrom)s_%(pop)s_r21_nr_fwd_phased_all.imported'
% locals())
genMapImportedFN = os.path.join(
projDir,
'genetic_map_chr%(chrom)s_%(genMapSfx)s.txt.imported' %
locals())
pr.addRule(
name='extractSnpInfo',
commands=
'java -classpath ../Other/Ilya_Other/sweep/sweepsrc/sweep.jar edu.mit.broad.sweep.Main ExtractAlleleFreqs %(projDir)s/project %(snpInfoFN)s %(pop)s %(chrom)s'
% locals(),
commandsOld=
'java -classpath ../Other/Ilya_Other/sweep/sweepsrc/sweep/target/sweep-1.0-SNAPSHOT-jar-with-dependencies.jar edu.mit.broad.sweep.Main ExtractAlleleFreqs %(projDir)s/project %(snpInfoFN)s %(pop)s %(chrom)s'
% locals(),
depends_on=(ancestralImportedFN, genotypesImportedFN,
genMapImportedFN),
creates=snpInfoFN)
chr2dihhFN = {}
for chrom in chroms:
with pr.settingAttrs('chrom'):
chrom_s = 'chr' + str(chrom)
chromDir = os.path.join(cmsDir, chrom_s)
xpopScoresFN = os.path.join(
chromDir, AddFileSfx('max_xpop.tsv', chrom_s, selPop, pops))
pr.addInvokeRule(invokeFn=gatherXPOPscores,
invokeArgs=Dict('pops chrom selPop sweepDir',
outFN=xpopScoresFN),
attrs=dict(pop=allPops,
stat='max_xpop',
piperun_short=True))
ihsFN = getFN_ihs_signif(**Dict('sweepDir chrom', pop=selPop))
ihsScoresFN = os.path.join(
chromDir, AddFileSfx('iHS.tsv', chrom_s, selPop, pops))
dihhScoresFN = os.path.join(
chromDir, AddFileSfx('dihh.tsv', chrom_s, selPop, pops))
chr2dihhFN[chrom] = dihhScoresFN
pop2ancFreqFN = os.path.join(
cmsDir, chrom_s, AddFileSfx('pop2ancFreq.tsv', chrom_s, pops))
pop2sampleSizeFN = os.path.join(
cmsDir, chrom_s, AddFileSfx('pop2sampleSize.tsv', chrom_s,
pops))
pop2snpInfoFN = dict([(pop,
os.path.join(sweepDir, 'analysis',
'chr%(chrom)s' % locals(),
'snps_%(pop)s.tsv' % locals()))
for pop in pops])
pr.addInvokeRule(
invokeFn=gather_snp_info,
invokeArgs=Dict(
'pops pop2snpInfoFN pop2ancFreqFN pop2sampleSizeFN'))
pr.addInvokeRule(
invokeFn=gather_iHS_scores,
invokeArgs=Dict(
'chrom selPop ihsFN pop2ancFreqFN',
# snpInfoFN = pop2snpInfoFN[ selPop ],
ihsOutFN=ihsScoresFN,
dihhOutFN=dihhScoresFN),
attrs=dict(pop=selPop,
stat=('iHS', 'StdDiff'),
piperun_short=True))
freqDiffScoresFN = os.path.join(
chromDir, AddFileSfx('freqDiff.tsv', chrom_s, selPop, pops))
meanFstScoresFN = os.path.join(
chromDir, AddFileSfx('meanFst.tsv', chrom_s, selPop, pops))
pr.addInvokeRule(
invokeFn=computeMeanFstAndFreqDiffScores,
invokeArgs=Dict(
'chrom selPop sweepDir pops pop2ancFreqFN pop2sampleSizeFN',
outMeanFstFN=meanFstScoresFN,
outFreqDiffFN=freqDiffScoresFN),
attrs=dict(pop=allPops,
stat=('freqDiff', 'meanFst'),
piperun_short=True))
StdDiffScoresFN = os.path.join(
chromDir, AddFileSfx('StdDiff.tsv', chrom_s, selPop, pops))
rawScoresFN[chrom] = dict(iHS=ihsScoresFN,
StdDiff=StdDiffScoresFN,
meanFst=meanFstScoresFN,
freqDiff=freqDiffScoresFN,
max_xpop=xpopScoresFN)
# end: with pr.settingAttrs( 'chrom' )
# end: for chrom in chroms
# ihhStdFN = os.path.join( cmsDir, 'dihhstd.tsv' )
dihhGlobalStdFN = os.path.join(
cmsDir, AddFileSfx('dihh_global_std.tsv', selPop, pops))
dihhBinMeansFN = os.path.join(
cmsDir, AddFileSfx('dihh_bin_means.tsv', selPop, pops))
pr.addInvokeRule(invokeFn=normalizeByFreq_getMeanStd_tsv,
invokeArgs=dict(
iHHDiffFNs=[chr2dihhFN[k] for k in chroms],
globalStatFN=dihhGlobalStdFN,
binsStatFN=dihhBinMeansFN),
name='compute_dihh_meanstd')
# pr.addInvokeRule( invokeFn = computeMeanStd_binned_tsvs,
# invokeArgs = dict( inFNs = chr2dihhFN.values(), valCol = 'iHHDiff',
# binCol = 'normingFreqs', binMin = 0.05, binMax = 1.05, binStep = .05,
# outFN = ihhStdFN ),
# name = 'compute_dihh_std' )
for chrom in chroms:
with pr.settingAttrs('chrom'):
chrom_s = 'chr' + str(chrom)
chromDir = os.path.join(cmsDir, chrom_s)
StdDiffScoresFN = os.path.join(
chromDir, AddFileSfx('StdDiff.tsv', chrom_s, selPop, pops))
dbg('chrom chr2dihhFN[chrom]')
pr.addInvokeRule(invokeFn=normalizeByFreq_compute_normed_tsv,
invokeArgs=dict(iHHDiffFN=chr2dihhFN[chrom],
globalStatFN=dihhGlobalStdFN,
binsStatFN=dihhBinMeansFN,
StdDiffFN=StdDiffScoresFN))
statLikesRatioFNs = {}
for stat in CMSBins.CMSstats:
with pr.settingAttrs(
stat=stat,
pop=(selPop, ) if stat in ('iHS', 'StdDiff') else allPops,
piperun_short=True):
if stat not in CMSBins.nonNormedStats:
rawFNs = [rawScoresFN[chrom][stat] for chrom in chroms]
meanStdFN = os.path.join(
cmsDir, AddFileSfx('meanStd.tsv', stat, selPop, pops))
# DefineRulesTo_computeMeanStd( pr, inFNs = rawFNs, colNum = 1,
# outFN = meanStdFN,
# addRuleArgs = \
# dict( name = 'computeMeanStd_for_stat',
# attrs = dict( chrom = chroms ) ) )
# meanStdBzFN = os.path.join( cmsDir, stat + '_meanStdForStat.tsv' )
pr.addInvokeRule(invokeFn=computeMeanStd,
invokeArgs=dict(inFNs=rawFNs,
colName=stat,
outFN=meanStdFN))
# end: if stat not in CMSBins.nonNormedStats
for chrom in chroms:
with pr.settingAttrs('chrom'):
statFN = rawScoresFN[chrom][stat]
if stat not in CMSBins.nonNormedStats:
normedFN = AddFileSfx(statFN, 'normed')
DefineRulesTo_normalizeOneColumn(
pr,
inFN=statFN,
meanStdFN=meanStdFN,
colName=stat,
outFN=normedFN,
addRuleArgs=dict(attrs=Dict('chrom')))
statFN = normedFN
bins_beg = CMSBins.stat_start[stat]
bins_end = CMSBins.stat_end[stat]
bins_n = CMSBins.stat_nbin[stat]
statLikesRatioFN = AddFileSfx(rawScoresFN[chrom][stat],
'likesRatio')
statLikesRatioFNs[(chrom, stat)] = statLikesRatioFN
pr.addInvokeRule(
invokeFn=computeLikeRatioForStat,
invokeArgs=dict(
stat=stat,
statValsFN=statFN,
hitLikesFN=
'../Data/Common_Data/sim/likes/hitsLikes_toneutFixed_1.tsv',
missLikesFN=
'../Data/Common_Data/sim/likes/missLikes_toneutFixed_1.tsv',
stat_start=bins_beg,
stat_end=bins_end,
stat_nbin=bins_n,
statLikesRatioFN=statLikesRatioFN))
# end: with pr.settingAttrs( 'chrom' )
# end: for chrom in chroms
# end: with pr.settingAttrs( stat = stat, piperun_short = True )
# end: for stat in CMSBins.CMSstats
for chrom in chroms:
with pr.settingAttrs(chrom=chrom, stat=CMSBins.CMSstats):
chrom_s = 'chr' + str(chrom)
chromDir = os.path.join(cmsDir, chrom_s)
likesRatioFN = os.path.join(
chromDir,
AddFileSfx('likesRatio.tsv', CMSBins.CMSstats, selPop, pops))
pr.addInvokeRule(invokeFn=addLikesRatios,
invokeArgs=dict(
inFNs=[
statLikesRatioFNs[(chrom, stat)]
for stat in CMSBins.CMSstats
],
colNames=[
colName + 'likeRatio'
for colName in CMSBins.CMSstats
],
outFN=likesRatioFN))
def mergeSims( scenario, Ddata, posFileFN = None, simsOut = 'simsOut', nreplicas = 100, thinExt = '', thinSfx = '',
putativeMutPop = None, outFile = None,
pop2name = pop2name, statsSfx = '', ihsSfx = '',
limitToPop = None,
getio = None ):
"""Gathers per-SNP information, for all replicas of a given scenario, and outputs it in a single DotData where each line
gives info for one SNP.
Specifically, reads simulation and Sweep output, collects columns needed for composite likehood test (chrom, base pair position, genetic
distance, anc frequencies for 3 populations, xpop for each pair, and ihs, iHH_A and iHH_D for selected population)
Input params:
scenario - an object of class Scenario, indicating the simulation scenario (either neutral or a selection scenario)
from which all replicas were simulated.
nreplicas - the number of replicas simulated under this scenario.
Each replica represents a chromosome region, with a set of SNPs on it.
Ddata - the directory under which the simulations and the Sweep analysis results live.
Under this directory we expect to find:
iHS analysis results, under power_ihs/
XP-EHH analysis results, under power_xpop
simulation output giving SNP positions
thinExt - the extension appended to simulation files that describe the SNPs in the simulated replica.
Sometimes we create simulations and then thin them under different thinning models (to simulate SNP ascertainment
by the various stages of HapMap; these differently thinned versions of the same simulations might be stored in
simulation files with different extensions.
thinSfx - the suffix appended to the power_ihs and power_xpop directory names, telling where to find iHS and XP-EHH
analyses of the simulations. When we analyze the same simulations after applying different thinning scenarios,
the iHS and XP-EHH analyses for each thinning scenario go into a separate set of directories.
putativeMutPop - the population in which, we think, selection is occurring, aka "putatively selected population".
In practice, when localizing a given region, we will usually suspect that selection has occurred in a particular
population. When doing a genome-wide scan, we can do several scans assuming each population in turn to be
the selected population, and find regions selected in that population.
Output params:
Ddata - under Ddata writes a DotData named merged_scenName.data, where each line gives info
for one SNP, with the following columns (type of data is float unless stated otherwise):
CHROM_POS 1 - physical (basepair) position of the SNP within its replica.
Note that one merged file contains SNPs for a set of replicas (all for the same scenario),
so there could be multiple SNPs with the same position. The replica number
is given in the Chrom column.
FREQ1 1 - derived allele frequency in pop 1 ( European )
FREQ1 4 - derived allele frequency in pop 4 ( EastAsian )
FREQ1 5 - derived allele frequency in pop 5 ( WestAfrican )
R AllEHH logratio Deviation European_WestAfrican - XP-EHH score to the right of the SNP,
between European and WestAfrican pops, normalized to the neutral background.
Analogously for the next five columns:
L AllEHH logratio Deviation European_WestAfrican
R AllEHH logratio Deviation EastAsian_European
L AllEHH logratio Deviation EastAsian_European
R AllEHH logratio Deviation EastAsian_WestAfrican
L AllEHH logratio Deviation EastAsian_WestAfrican
SNP pos (cM) European_WestAfrican - genetic map position of this SNP, within its replica.
(the European_WestAfrican suffix is irrelevant).
SNP pos (bases) European_WestAfrican - physical (basepair) position of this SNP within its replica.
(the European_WestAfrican suffix is irrelevant).
Chrom European_WestAfrican - the replica from which this SNP comes; can be nan.
(the European_WestAfrican suffix is irrelevant)
Chrom - the replica from which this SNP comes; can be nan
SNP pos (bases) - physical (basepair) position of this SNP within its replica.
SNP pos (cM) - genetic map position of this SNP within its replica
Both iHH_A - sum of iHH_A for both directions from this SNP
Both iHH_D - sum of iHH_D for both directions from this SNP
Both iHS - the value in 'Both Unstandardised iHS' (below), but binned by derived allele frequency
and normalized within the bin.
Left iHH_D - iHH_D to the left of the SNP (the raw integral value). analogously for the next three.
Right iHH_D
Left iHH_A
Right iHH_A
Both Unstandardised iHS - log( (iHH_A_left + iHH_A_right) / ( iHH_D_left + iHH_D_right ) )
( see also 'Both iHS' column for the standardized iHS score )
"""
if not Ddata.endswith('/'): Ddata += '/'
assert nreplicas > 0
dbg( 'pop2name' )
SimDir = os.path.join( Ddata, simsOut + thinSfx )
scenName = scenario.scenName()
scenDir = scenario.scenDir()
if putativeMutPop == None: putativeMutPop = scenario.mutPop
ihsSignifFN = os.path.join( Ddata, 'power_ihs' + thinSfx, scenDir,
'ihs_sig_' + pop2name[ putativeMutPop ] + ihsSfx + '.tsv' )
popNames = sorted( pop2name.values() )
popNums = sorted( pop2name.keys() )
minPopNum = popNums[ 0 ]
posFileKeyCols = ( 'replicaNum', 'CHROM_POS %d' % minPopNum )
xpopIhsKeyCols = ('Chrom', 'SNP pos (bases)')
popPairs = [ '%s_%s' % ( popNames[ pop1idx ], popNames[ pop2idx ] )
for pop1idx in range( len( popNames ) ) for pop2idx in range( pop1idx+1, len( popNames ) )
if limitToPop is None or limitToPop in ( popNames[ pop1idx ], popNames[ pop2idx ] ) ]
xpopSignifFNs = [ os.path.join( Ddata, 'power_xpop' + thinSfx, scenDir, 'xpop_significance_' + popPair + '.tsv' )
for popPair in popPairs ]
snpStatsDir = os.path.join( Ddata, 'snpStats' + thinSfx, scenario.scenDir() )
mergedData = outFile if outFile else os.path.join( snpStatsDir, AddFileSfx( 'merged.tsv', statsSfx, putativeMutPop, ihsSfx ) )
fileDescrs = \
{ mergedData :
( 'Various per-snp statistics for SNPs in scenario $scenario, replicas 0-$nreplicas, '
'assuming selection in ' + pop2name[ putativeMutPop ],
( ( 'CHROM_POS 1', 'physical (basepair) position of the SNP within its replica. '
'Note that one merged file contains SNPs for a set of replicas (all for the same scenario), '
'so there could be multiple SNPs with the same position. The replica number '
'is given in the Chrom column. ' ),
( 'FREQ1 1', 'derived allele frequency in pop 1 ( European )' ),
( 'R AllEHH logratio Deviation European_WestAfrican', 'XP-EHH score to the R of the SNP, '
'between European and WestAfrican pops, normalized to the neutral background.' ),
( 'SNP pos (cM) European_WestAfrican', 'genetic map SNP position' ),
( 'SNP pos (bases) European_WestAfrican', 'physical SNP position' ),
( 'Chrom European_WestAfrican', 'chromosome (or replica number)' ),
( 'Chrom', 'chromosome (or replica number)' ),
( 'SNP pos (bases)', 'physical SNP position' ),
( 'SNP pos (cM)', 'genetic map SNP position' ),
( 'Both iHH_A', 'sum of iHH_A scores for both sides' ),
( 'Both iHH_D', 'sum of iHH_D scores for both sides' ),
( 'Both iHS', 'sum of iHS scores for both sides' ),
( ' Left iHH_D', 'iHH_D score to the left of the SNP' ),
( 'Right iHH_D', 'iHH_D score to the right of the SNP' ),
( 'Left iHH_A', 'iHH_A score to the left of the SNP' ),
( 'Right iHH_A', 'iHH_A score to the right of the SNP' ),
( 'Both Unstandardised iHS', 'sum of unstandardized iHS scores for both sides' ) ) ) }
if posFileFN is None: posFileFN = os.path.join( Ddata, 'snpStats' + thinSfx, scenario.scenDir(),
AddFileSfx( 'mergedPosStacked.tsv', statsSfx, putativeMutPop, ihsSfx ) )
if getio: return dict( depends_on = [ posFileFN, ihsSignifFN ] + xpopSignifFNs, creates = mergedData,
splitByCols = dict([ ( posFileFN, dict( keyCols = posFileKeyCols ) ) ]
+ [ ( signifFN, dict( keyCols = xpopIhsKeyCols ) ) for signifFN in [ ihsSignifFN ] + xpopSignifFNs ] ),
mediumRuleNameSfx = ( scenario.scenDir(), putativeMutPop ),
fileDescrs = fileDescrs,
attrs = Dict( 'putativeMutPop nreplicas pop2name' ) )
dashFixer = lambda v: v if v != '-' else np.nan
ihsAll = IDotData(ihsSignifFN, valueFixer = dashFixer)
ihsAll = ihsAll[('Chrom','SNP pos (bases)','SNP pos (cM)','Both iHH_A','Both iHH_D','Both iHS',
'Left iHH_D','Right iHH_D','Left iHH_A','Right iHH_A','Both Unstandardised iHS')]
def chkReplica( r, n = nreplicas ): return r.Chrom < n
ihsAll = ihsAll.takewhile( chkReplica )
xpopCols = ('Chrom','SNP pos (bases)','SNP pos (cM)','L AllEHH logratio Deviation','R AllEHH logratio Deviation')
xpopSignif = tuple( [ IDotData(xpopSignifFN, valueFixer = dashFixer)[ xpopCols].takewhile( chkReplica )
for xpopSignifFN in xpopSignifFNs ] )
posCols = ['CHROM_POS %d' % minPopNum ] + ['FREQ1 %d' % popNum for popNum in popNums]
result = IDotData.merge( iDotDatas = ( IDotData( posFileFN ), ) + xpopSignif + ( ihsAll, ),
cols = (posFileKeyCols,) +
(xpopIhsKeyCols,) * ( len( popPairs ) + 1 ),
blanks = (None,) + (np.nan,) * ( len( popPairs ) + 1 ),
suffixes = ['pos'] + [ ' %s' % popPair for popPair in popPairs ] + [ '' ] )
aPopPair = 'European_WestAfrican' if 'European_WestAfrican' in popPairs else popPairs[0]
useCols = [ 'replicaNum' ] + posCols + \
[ '%s AllEHH logratio Deviation %s' % ( side, popPair ) for popPair in popPairs for side in ( 'L', 'R' ) ] + \
[ 'SNP pos (cM) ' + aPopPair,
'SNP pos (bases) ' + aPopPair,
'Chrom ' + aPopPair,
'Chrom',
'SNP pos (bases)',
'SNP pos (cM)',
'Both iHH_A',
'Both iHH_D',
'Both iHS' ]
if len( popPairs ) == 1:
result = result.renameCols( { 'L AllEHH logratio Deviation' : 'L AllEHH logratio Deviation ' + aPopPair,
'R AllEHH logratio Deviation' : 'R AllEHH logratio Deviation ' + aPopPair } )
result[ useCols ].save( mergedData )
logging.info( 'Finished mergeSims()' )
def runCmdParallelized(commands,
depends_on,
creates,
comment,
splitFunc,
joinFunc,
saveOutputTo=None,
splitFN=None,
joinFN=None,
name=None,
mediumRuleName=None,
getio=None):
"""Run the specified command, using parallelization."""
from Operations.Ilya_Operations.PipeRun.python.PipeRun import PipeRun
dbg('"IN_RUNCMDPAR_EEEEEE" depends_on creates saveOutputTo')
if creates is None: creates = ()
commands = MakeSeq(commands)
depends_on = MakeSeq(depends_on)
creates = MakeSeq(creates)
gio = Dict('depends_on creates comment name mediumRuleName')
dbg('gio')
if getio:
return Dict(
'depends_on creates comment name mediumRuleName saveOutputTo',
uses=(splitFunc, joinFunc))
splitFN = splitFN or list(MakeSeq(depends_on))[0]
joinFN = RandomString(12) if saveOutputTo else (
joinFN or list(MakeSeq(creates))[0])
assert any([splitFN in command for command in commands])
assert saveOutputTo or any([joinFN in command for command in commands])
logging.info('calling ' + str(splitFunc) + ' to split ' + splitFN)
outDir = os.path.join('/broad/hptmp', getpass.getuser(), 'par',
os.path.abspath(splitFN)[1:])
pr = PipeRun(name='splitting', descr='splitting')
r = pr.addInvokeRule(invokeFn=doSplit,
invokeArgs=Dict('splitFunc splitFN outDir'))
pr.runSubPipeline()
chunkFNs = SlurpFileLines(r.creates[0])
logging.info('finished running ' + str(splitFunc) + ' to split ' + splitFN)
dbg('"CHUNKS_ARE" chunkFNs')
pr = PipeRun(name='parallelizing', descr='parallelizing')
chunkOutFNs = []
for chunkFN in chunkFNs:
chunkOutFN = AddFileSfx(chunkFN, 'out')
chunkOutFNs.append(chunkOutFN)
for command in commands:
dbg('splitFN chunkFN chunkOutFN command command.replace(splitFN,chunkFN)'
)
pr.addRule(
commands=[
command.replace(splitFN, chunkFN).replace(joinFN, chunkOutFN)
for command in commands
],
depends_on=[f if f != splitFN else chunkFN for f in depends_on],
creates=[f if f != joinFN else chunkOutFN for f in creates],
saveOutputTo=None if saveOutputTo is None else chunkOutFN)
pr.runSubPipeline()
joinFunc(inFNs=chunkOutFNs, outFN=None if saveOutputTo else joinFN)
def localizeSpatiallyByWindows(Ddata,
scenario,
nreplicas,
thinSfx='',
putativeMutPop=None,
complikeSfx='',
likesTableSfx='',
threshold=.5,
numSNP=1,
minGdInEachDir=.05,
fromReplica=None,
toReplica=None,
getio=None):
"""
Spatially localize the selected variant for all replicas within a given scenario.
The approach is to start with the highest-scoring SNP, and move left and right from it in fixed-size windows
for as long as the windows contain at least 'numSNP' snps with score at least 'threshold'.
Adapted from Operations.Shari_Operations.localize.hapmap_regions_0615.plotRegions() .
"""
snpStatsDir = os.path.join(Ddata, 'snpStats' + thinSfx, scenario.scenDir())
replicaStatsDir = os.path.join(Ddata, 'replicastats' + thinSfx,
scenario.scenDir())
if putativeMutPop == None: putativeMutPop = scenario.mutPop
sfxs = (putativeMutPop, complikeSfx, likesTableSfx)
complikeFN = os.path.join(snpStatsDir, AddFileSfx('complike.data/', *sfxs))
intervalsListFN = os.path.join(
replicaStatsDir, AddFileSfx('intervalsWindowsList.tsv', *sfxs))
if getio:
return dict(
depends_on=complikeFN,
creates=intervalsListFN,
mediumRuleNameSfx=(scenario.scenDir(), ) + sfxs,
fileDescrs={
intervalsListFN:
'List of intervals in the region, one of which (hopefully) contains the causal SNP.'
' For each replica this table has one or more lines, giving intervals in that replica.'
})
#complike = IDotData( complikeFN ).filter( lambda r: all( np.isfinite( ( r.iHS, r.meanFst, r.max_xpop ) ) ) )
complike = IDotData(complikeFN)
with IDotData.openForWrite(
intervalsListFN,
'replicaNum gdFrom gdTo gdSize bpFrom bpTo bpSize numPositiveBins '
'numSnpsOver_0_2 maxSNP_Pos maxSNP_lik') as intervalsListFile:
for replicaNum, complikeForReplica in complike.groupby('Chrom'):
if fromReplica is not None and replicaNum < fromReplica: continue
if toReplica is not None and replicaNum > toReplica: break
X = complikeForReplica.toDotData()
minPos = np.min(X.gdPos)
maxPos = np.max(X.gdPos)
bins = np.arange(0, 1, .01)
ind = X.complikeExp.argsort()
lik = X.complikeExp[ind]
maxlik = np.mean(lik[-5:])
#maxlik = mean(lik[-5:])
like = X.complikeExp / maxlik
Y = X[ind]
maxSNP = Y.Pos[-1]
maxScore = Y.complikeExp[-1]
relPos = X.gdPos - Y.gdPos[-1]
X = X.hstack(
DotData(Columns=[like, relPos],
names=['scaled_like', 'relPos']))
topGdPos = Y.gdPos[-1]
minPos = Y.Pos[-1]
maxPos = Y.Pos[-1]
minGdPos = topGdPos
maxGdPos = topGdPos
numPositiveBins = 0
dbg('replicaNum minPos maxPos minGdPos maxGdPos')
for dir in -1, +1:
for bin in bins:
Z = X[np.abs(X.relPos - dir * bin) <= .02]
dbg('dir bin len(Z)')
if len(Z) == 0: continue
top = Z[Z.scaled_like > threshold]
dbg('len(top)')
if len(top) <= numSNP and np.abs(
topGdPos - top.gdPos) > minGdInEachDir:
break
if len(top) == 0: top = Z
if dir == -1:
minPos = np.min(top.Pos)
minGdPos = np.min(top.gdPos)
else:
maxPos = np.max(top.Pos)
maxGdPos = np.max(top.gdPos)
numPositiveBins += 1
ind = np.all([X.Pos > minPos, X.Pos < maxPos], axis=0)
peak = X[ind]
intervalsListFile.writeRecord(replicaNum, minGdPos, maxGdPos,
maxGdPos - minGdPos, minPos, maxPos,
maxPos - minPos, numPositiveBins,
sum(peak.scaled_like > .2), maxSNP,
maxScore)
def evalSpatialLoc(Ddata,
thinSfx,
scenario,
putativeMutPop,
nreplicas,
complikeSfx='',
likesTableSfx='',
selpos=500000,
whichSpatialLoc='Spline',
getio=None):
"""Evaluate spatial localization. Compute relevant replica statistic. For each replica,
compute: whether the localized intervals include the causal SNP; statistics about the
localized intervals; the position of the causal SNP relative to the localized intervals."""
assert not scenario.isNeutral()
snpStatsDir = os.path.join(Ddata, 'snpStats' + thinSfx, scenario.scenDir())
replicaStatsDir = os.path.join(Ddata, 'replicastats' + thinSfx,
scenario.scenDir())
if putativeMutPop == None: putativeMutPop = scenario.mutPop
sfxs = (putativeMutPop, complikeSfx, likesTableSfx)
intervalsListFN = os.path.join(
replicaStatsDir,
AddFileSfx('intervals%sList.tsv' % whichSpatialLoc, *sfxs))
causalGdPosFN = os.path.join(replicaStatsDir, 'causalGdPos.tsv')
spatialLocEvalFN = os.path.join(
replicaStatsDir,
AddFileSfx('spatialLocEval%s.tsv' % whichSpatialLoc, *sfxs))
if getio:
return dict(depends_on=(intervalsListFN, causalGdPosFN),
creates=spatialLocEvalFN,
mediumRuleNameSfx=scenario.scenDir(),
name='evalSpatialLoc_' + whichSpatialLoc)
with IDotData.openForWrite(
spatialLocEvalFN,
'replicaNum numIntervals totLenBp totLenGd causalIncluded '
'distanceToIntervalBoundaryBp distanceToIntervalBoundaryGd'
) as spatialLocEvalFile:
for ( replicaNum2, replicaIntervals ), ( replicaNum3, causalGdPos, replicaNum4 ) in \
zip( IDotData( intervalsListFN ).groupby( 'replicaNum' ),
IDotData.merge( iDotDatas = ( IDotData( causalGdPosFN ),
IDotData( intervalsListFN ).replicaNum.removeDups() ),
cols = ( 'replicaNum', 'replicaNum' ) ) ):
replicaNum2, replicaNum3, replicaNum4 = list(
map(int, (replicaNum2, replicaNum3, replicaNum4)))
if not replicaNum2 == replicaNum3 == replicaNum4:
dbg('replicaNum2 replicaNum3 replicaNum4 intervalsListFN complikeFN causalGdPosFN spatialLocEvalFN'
)
assert replicaNum2 == replicaNum3 == replicaNum4
causalIncluded = False
totLenBp = 0
totLenGd = 0.0
for replicaInterval in replicaIntervals:
dbg('replicaInterval causalGdPos')
#assert bool( replicaInterval.bpFrom <= selpos <= replicaInterval.bpTo ) == bool( replicaInterval.gdFrom <= causalPos_gd <= replicaInterval.gdTo )
assert (replicaInterval.gdFrom <= causalGdPos <=
replicaInterval.gdTo) == (replicaInterval.bpFrom <=
selpos <=
replicaInterval.bpTo)
if replicaInterval.gdFrom <= causalGdPos <= replicaInterval.gdTo:
causalIncluded = True
totLenGd += (replicaInterval.gdTo - replicaInterval.gdFrom)
totLenBp += (replicaInterval.bpTo - replicaInterval.bpFrom)
spatialLocEvalFile.writeRecord(
replicaNum2, len(replicaIntervals), totLenBp, totLenGd,
int(causalIncluded),
np.min((np.min(np.abs(replicaIntervals.bpFrom - selpos)),
np.min(np.abs(replicaIntervals.bpTo - selpos)))),
np.min((np.min(np.abs(replicaIntervals.gdFrom - causalGdPos)),
np.min(np.abs(replicaIntervals.gdTo - causalGdPos)))))
def localizeSpatiallyBySplineFitting(Ddata,
scenario,
nreplicas,
thinSfx='',
putativeMutPop=None,
complikeSfx='',
likesTableSfx='',
confidence=.9,
minBins=20,
nbins=200,
smoothing=0.0,
getio=None):
"""For each replica within a given scenario,
localize the selected SNP spatially, by fitting a spline to a (smoothed version of) the CMS scores, dividing the
region into bins, and finding the set of bins that cover 90% (or specified fraction of) area under the spline.
Params:
confidence - the spatially localized region will (hopefully) have this probability of containing the causal SNP;
here, specifically, this means we'll include bins in the region that collectively cover this fraction of area
under the posterior density curve.
minBins - the region will include at least this many of the highest-average bins.
"""
snpStatsDir = os.path.join(Ddata, 'snpStats' + thinSfx, scenario.scenDir())
replicaStatsDir = os.path.join(Ddata, 'replicastats' + thinSfx,
scenario.scenDir())
if putativeMutPop == None: putativeMutPop = scenario.mutPop
sfxs = (putativeMutPop, complikeSfx, likesTableSfx)
complikeFN = os.path.join(snpStatsDir, AddFileSfx('complike.data/', *sfxs))
intervalsListFN = os.path.join(
replicaStatsDir, AddFileSfx('intervalsSplineList.tsv', *sfxs))
intervalsStatsFN = os.path.join(
replicaStatsDir, AddFileSfx('intervalsSplineStats.tsv', *sfxs))
posteriorSplineFN = os.path.join(
replicaStatsDir, AddFileSfx('intervalsSplineSpline.tsv', *sfxs))
binInfoFN = os.path.join(replicaStatsDir,
AddFileSfx('intervalsSplineBinInfo.tsv', *sfxs))
if getio:
return dict(
depends_on=complikeFN,
creates=(intervalsListFN, intervalsStatsFN, posteriorSplineFN,
binInfoFN),
mediumRuleNameSfx=(scenario.scenDir(), ) + sfxs,
fileDescrs={
intervalsListFN:
'List of intervals in the region, one of which (hopefully) contains the causal SNP.'
' For each replica this table has one or more lines, giving intervals in that replica.',
intervalsStatsFN:
'Per-replica statistic about confidence intervals in that replica',
posteriorSplineFN:
'The results of interpolating posterior density as a spline',
binInfoFN:
'Information about individual bins under the spline'
})
complike = IDotData(complikeFN).filter(
lambda r: all(np.isfinite((r.iHS, r.meanFst, r.max_xpop))))
binInfoHeadings = 'replicaNum binNum binStart binEnd included binCenters binAvgCMS binMaxCMS binIntegral '\
'binIntegralNormed binRank'
nbins_orig = nbins
with contextlib.nested( IDotData.openForWrite( posteriorSplineFN, 'replicaNum gdPos complikeExp' ),
IDotData.openForWrite( binInfoFN, binInfoHeadings ),
IDotData.openForWrite( intervalsListFN, 'replicaNum gdFrom gdTo gdSize bpFrom bpTo '
'bpSize binsInInterval binsArea binsMax binsMaxFrac binsAvg '
'binsMinRank binsMaxRank' ),
IDotData.openForWrite( intervalsStatsFN, 'replicaNum numSegments gdTotLen bpTotLen' ) ) \
as ( posteriorSplineFile, binInfoFile, intervalsListFile, intervalsStatsFile ):
for replicaNum, complikeForReplica in complike.groupby('Chrom'):
gdMin = np.min(complikeForReplica.gdPos)
gdMax = np.max(complikeForReplica.gdPos)
binSize = (gdMax - gdMin) / nbins_orig
binLefts = np.arange(gdMin, gdMax, binSize)
binCenters = binLefts + binSize / 2
binRights = binLefts + binSize
dbg('len(complikeForReplica) replicaNum len(binLefts) len(binRights) len(binCenters)'
)
nbins = len(binLefts)
#
# Compute the mean and max CMS score in each gdPos bin
#
binAvgCMS = np.zeros(nbins)
binMaxCMS = np.zeros(nbins)
binNums = np.zeros(nbins, dtype=int)
for bin, valsInBin in \
complikeForReplica.addCol( 'bin',
list(map( functools.partial( min, nbins-1 ),
list(map( int,
( complikeForReplica.gdPos - gdMin ) / binSize ))))).groupby('bin'):
binNums[bin] = bin
if valsInBin:
binAvgCMS[bin] = np.mean(valsInBin.complikeExp)
binMaxCMS[bin] = max(valsInBin.complikeExp)
else:
binAvgCMS[bin] = binAvgCMS[bin - 1]
binMaxCMS[bin] = binMaxCMS[bin - 1]
# fit a spline to the function ( binCenter, binAvgCMS ), approximating the posterior probability of a SNP being
# causal.
posteriorDensitySpline = interpolate.splrep(binCenters,
binAvgCMS,
s=smoothing)
splineX = np.arange(gdMin, gdMax, binSize / 2)
for x, y in zip(splineX,
interpolate.splev(splineX,
posteriorDensitySpline)):
posteriorSplineFile.writeRecord(replicaNum, x, y)
# compute the integral under the interpolated function, for each bin.
binIntegral = np.zeros(nbins)
for binLeft, binRight, binNum in zip(binLefts, binRights, binNums):
binIntegral[binNum] = interpolate.splint(
binLeft, binRight, posteriorDensitySpline)
# normalize the integral value above each bin so that the total posterior density of being causal
# integrates to 1.0
binIntegralNormed = binIntegral / sum(binIntegral)
binsByIntegralSize = binIntegral.argsort()[::-1]
binRank = np.zeros(nbins)
for i in range(nbins):
binRank[binsByIntegralSize[i]] = i
binsToUse = np.zeros(nbins, dtype=bool)
binsIncluded = 0
fractionCovered = 0.0
for bin in binsByIntegralSize:
if fractionCovered >= confidence and binsIncluded >= minBins:
break
binsToUse[bin] = True
binsIncluded += 1
fractionCovered += binIntegralNormed[bin]
# list the confidence intervals for this replica,
# merging adjacent intervals.
gd2bp = interpolate.interp1d(complikeForReplica.gdPos,
complikeForReplica.Pos)
gdMax = max(complikeForReplica.gdPos)
binInfo = IDotData(names=binInfoHeadings,
Columns=(itertools.repeat(replicaNum, nbins),
list(range(nbins)), binLefts,
binRights, binsToUse, binCenters,
binAvgCMS, binMaxCMS, binIntegral,
binIntegralNormed, binRank))
binInfoFile.writeRecords(binInfo)
binsOverallMaxCMS = max(binInfo.binMaxCMS)
gdTotLen = 0.0
bpTotLen = 0
numSegments = 0
for included, bins in binInfo.groupby('included'):
if included:
gdFrom = min(bins.binStart)
gdTo = min(max(bins.binEnd), gdMax)
bpFrom = int(gd2bp(gdFrom))
bpTo = int(gd2bp(gdTo))
gdTotLen += (gdTo - gdFrom)
bpTotLen += (bpTo - bpFrom)
numSegments += 1
intervalsListFile.writeRecord(
replicaNum, gdFrom, gdTo, gdTo - gdFrom, bpFrom, bpTo,
bpTo - bpFrom, len(bins), sum(bins.binIntegralNormed),
max(bins.binMaxCMS),
max(bins.binMaxCMS) / binsOverallMaxCMS,
np.mean(bins.binAvgCMS), min(bins.binRank),
max(bins.binRank))
assert numSegments > 0 and bpTotLen > 0 and gdTotLen > 0.0
intervalsStatsFile.writeRecord(replicaNum, numSegments, gdTotLen,
bpTotLen)
dbg('replicaNum numSegments gdTotLen bpTotLen')
def DefineRulesTo_RunSweepOnSims(pr,
Ddata,
simsOut,
thinExt='',
thinning='',
suffix='',
mutAges=AllAges,
mutPops=AllPops,
mutFreqs=AllFreqs,
nreplicas=100,
pop2name=pop2name,
tests=('lrh', 'ihs', 'xpop'),
acceptExistingSimConfigFiles=False,
setOptions=(),
appendOptions=(),
inputParamsFiles=[],
runImportsLocally=True,
noImports=False,
powerSfx='',
doOnlyStages=None):
"""Define rules for running simulations and doing Sweep analyses of them.
Parameters:
mutAges, mutPops, mutFreqs - parameters defining the selection scenario.
"""
# Define the rules to do Sweep analyses of the simulations.
# These rules are created by running a Perl script, sim_analysis_pipe.pl .
# Each invocation of the script defines rules for one type of test.
# The rules for each test are saved to an .xml file, and then all these files are
# merged into pr.
# Rather than explicitly invoking the script three times we define a small pipeline to do this.
# The output of this pipeline is a pipeline definition for analyzing simulation results by each of the tests,
# saved to an .xml file.
simPipeline = PipeRun(name='DefineSims',
descr='Define simulation and analysis pipeline')
if not powerSfx: powerSfx = Sfx(*mutAges)
if not acceptExistingSimConfigFiles:
simPipeline.addInvokeRule(
invokeFn=WriteSimulationInfo,
invokeArgs=Dict('Ddata mutAges mutPops mutFreqs nreplicas suffix '
'inputParamsFiles pop2name powerSfx'))
test2pipeline = {}
for test in tests:
simSfx = ''
if suffix: simSfx += suffix
if thinning: simSfx += thinning
thinExtHere = '' if thinning else thinExt
testPipeline = os.path.join('Ilya_Temp',
AddFileSfx('p.xml', pr.name, test))
test2pipeline[test] = testPipeline
if thinning:
simPipeline.addRule(
comment="Copy config file",
targets="$Ddata/power_$test$simSfx/config$powerSfx.txt",
sources="$Ddata/power_$test$suffix/config.txt",
commands="cp $Ddata/power_$test$suffix/config.txt "
"$Ddata/power_$test$simSfx/config$powerSfx.txt",
mediumRuleName='copy_config_' + test,
name='copy_config')
simPipeline.addRule(
targets=testPipeline,
sources=[
'../Other/Ilya_Other/sweep/sims/scripts/sim_analysis_pipe.pl',
"$Ddata/power_$test$simSfx/config$powerSfx.txt",
"$Ddata/config$suffix/sims$powerSfx.txt",
"$Ddata/config$suffix/scenarios$powerSfx.txt",
"$Ddata/config$suffix/pops$powerSfx.txt"
],
commands=
"../Other/Ilya_Other/sweep/sims/scripts/sim_analysis_pipe.pl "
" --only-write-pipeline $testPipeline " +
('--run-imports-locally ' if runImportsLocally else '') +
('--no-imports ' if noImports else '') +
(('--do-only-stages ' + doOnlyStages +
' ') if doOnlyStages else '') +
('--sim-suffix ' + simSfx if simSfx else '') +
('--powerSfx ' + powerSfx if powerSfx else '') +
(' --thin-ext ' + thinExtHere if thinExtHere else '') +
reduce(operator.concat, [
' --set-option ' + setOption[0] + ' ' + setOption[1]
for setOption in setOptions
], '') + reduce(operator.concat, [
' --append-option ' + appendOption[0] + " '" +
appendOption[1] + "'" for appendOption in appendOptions
], '') +
" --target-test $test $Ddata/$simsOut$simSfx $Ddata/power_$test$simSfx",
name='simanal',
mediumRuleName='simanal_$test$simSfx',
comment='Define rules for analyzing simulations')
dbg('"RUNNING_simPipeline"')
simPipeline.runForced(aftRuleDelay=5)
dbg('"DONE_RUNNING_simPipeline"')
for test in tests:
pr.addPipelineFromFile(test2pipeline[test])