def StartPushers(use_args=None, as_daemons=False):
parser = optparse.OptionParser()
parser.add_option('-P',
'--num-result-pushers',
type='int',
dest='numResultPushers',
help='create NUMPROCS parallel result pushers',
metavar='NUMPROCS',
default=1)
parser.add_option('-Q',
'--queues',
dest='queues',
default=os.path.join('..', 'Other', 'queues',
getpass.getuser()),
help='push results for the specified QUEUES',
metavar='QUEUES')
parser.add_option('-S',
'--sleepInterval',
type='int',
help='between checks, sleep for SEC seconds',
metavar='SEC',
default=20)
dbg('use_args')
(options, args) = parser.parse_args(
args=sys.argv[1:] if use_args is None else list(use_args))
dbg('options args')
assert not args
for i in range(options.numResultPushers):
p = multiprocessing.Process(target=PushResults, args=(options, ))
allPushers.append(p)
p.daemon = as_daemons
p.start()
time.sleep(min(1.0, random.normalvariate(4.0, 1.0)))
def computeLikeRatioForStat_do(statVals, hitLikes, missLikes, bins):
"""Compute likes ratio"""
# Precompute the likelihood ratio corresponding to each bin
dbg('statVals hitLikes missLikes bins')
indNaN = hitLikes != 1e-10
missingVal = np.log(np.min(hitLikes[indNaN] / missLikes[indNaN]))
CLR = [(np.log(hitLike / missLike) if hitLike != 1e-10 else missingVal)
if hitLike != 0.0 else np.nan
for hitLike, missLike in zip(hitLikes, missLikes)]
CLR = np.array([CLR[0]] + CLR + [CLR[-1]])
binIds = np.digitize(statVals.values, bins)
st_binSize = (bins[1] - bins[0])
st_nbins = len(bins) - 1
binIds2 = np.where(
np.isfinite(statVals.values),
np.clip(((statVals.values - bins[0]) / st_binSize).astype(np.int16), 0,
st_nbins - 1), len(hitLikes)) + 1
return np.where(np.isnan(statVals.values),
np.repeat(np.nan,
len(statVals)), CLR[binIds]), binIds, binIds2
def computeMeanFstAndFreqDiffScores(pops,
chrom,
selPop,
sweepDir,
pop2ancFreqFN,
pop2sampleSizeFN,
outMeanFstFN,
outFreqDiffFN,
getio=None):
"""Compute meanFst and freqDiff scores"""
if selPop not in pops: pops = tuple(MakeSeq(pops)) + (selPop, )
cmpPops = [pop for pop in pops if pop != selPop]
if getio:
return dict(depends_on=(pop2ancFreqFN, pop2sampleSizeFN),
creates=(outMeanFstFN, outFreqDiffFN),
attrs=Dict('chrom', pop=pops))
# pop2ancFreq.to_csv( 'befdrop.tsv', sep = '\t' )
# pop2ancFreq.fillna( value = 1.0, inplace = True )
# pop2ancFreq.to_csv( 'aftdrop.tsv', sep = '\t' )
pop2ancFreq = pd.read_table(pop2ancFreqFN, index_col='pos')
pop2sampleSize = pd.read_table(pop2sampleSizeFN,
index_col='pop').sampleSize
dbg('pop2sampleSize')
#pop2snpInfo.to_csv( 'test.tsv', sep = '\t', header = True )
derFreq = 1.0 - pop2ancFreq[selPop]
cmpAncFreqs = pop2ancFreq[[pop for pop in pops if pop != selPop]]
meanAnc = cmpAncFreqs.mean(axis=1)
freqDiff = derFreq - (1.0 - meanAnc)
freqDiff.name = 'freqDiff'
freqDiff.to_csv(outFreqDiffFN, sep='\t', header=True)
# compute meanFst
# dbg( '"vvvvvvvvvvvw" selPop pop2ancFreq[selPop] pop2ancFreq["JPT+CHB"] pop2ancFreq["YRI"]' )
# dbg( 'selPop pop2sampleSize[selPop] pop2sampleSize["JPT+CHB"] pop2sampleSize["YRI"]' )
d = dict([(pop,
fst_onePopPair(ancFreqs=np.array(
(pop2ancFreq[selPop], pop2ancFreq[pop])),
sampleSizes=(pop2sampleSize[selPop],
pop2sampleSize[pop])))
for pop in cmpPops])
fstVals = pd.DataFrame(data=d, index=pop2ancFreq.index)
# spc = fst_onePopPair( ancFreqs = np.array( ( pop2ancFreq[ 'BEB' ], pop2ancFreq[ 'ASN' ] ) ),
# sampleSizes = ( pop2sampleSize[ 'BEB' ], pop2sampleSize[ 'ASN' ] ) )
# dbg( '"ddddddddddd" fstVals.loc[526736] spc' )
# dbg( 'fstVals' )
fstVals.fillna(value=0.0, inplace=True)
#fstVals.to_csv( 'fstvals.tsv', sep = '\t', header = True, na_rep = 'NaN' )
fstMean = fstVals.mean(axis=1)
dbg('fstVals fstMean')
fstMean.name = 'meanFst'
fstMean.to_csv(outMeanFstFN, sep='\t', header=True, na_rep='NaN')
def LoadComparison(pop):
"""Load comparison with one pop"""
d0 = pd.read_csv(pop2FN[pop],
sep='\t',
usecols=('Pop 1', 'Pop 2', 'Chrom'),
nrows=1)
dbg('d0')
assert str(d0.loc[0, 'Chrom']) == str(chrom)
assert (d0.loc[0, 'Pop 1'] == selPop and d0.loc[0, 'Pop 2']
== pop) or (d0.loc[0, 'Pop 1'] == pop
and d0.loc[0, 'Pop 2'] == selPop)
flip = (d0.loc[0, 'Pop 1'] == pop)
d = pd.read_csv(pop2FN[pop],
sep='\t',
usecols=('SNP pos (bases)',
'L AllEHH logratio Deviation',
'R AllEHH logratio Deviation'),
index_col='SNP pos (bases)',
na_values=('-', ))
d.info()
if flip:
d['L AllEHH logratio Deviation'] *= -1
d['R AllEHH logratio Deviation'] *= -1
return pd.DataFrame.from_dict({pop: d.max(axis=1)})
def normalizeInBins_tsv(inDataFN,
valCol,
binCol,
binMin,
binMax,
binStep,
binsFN,
outFN,
normedCol,
getio=None):
"""Normalize data within bins, using previously computed bin means"""
if getio:
return dict(depends_on=(inDataFN, binsFN),
creates=outFN,
uses=normalizeInBins)
inData = pd.read_table(inDataFN)
binStats = pd.read_table(binsFN)
binMeans = binStats.means
totCount = float(binStats.counts.sum())
totMean = binStats.sums.sum() / totCount
commonStd = np.sqrt(binStats.sumsSq.sum() / totCount - totMean * totMean)
dbg('"CCCCCCCC" commonStd binMeans totCount totMean binStats.sums.sum() binStats.sumsSq.sum()'
)
normed = normalizeInBins(**Dict(
'inData valCol binCol binMin binMax binStep binMeans commonStd'))
inData.insert(len(inData.columns), normedCol, normed)
inData.to_csv(outFN, sep='\t', na_rep='NaN', index=False)
def DefineRulesTo_CreateSimulationParams(
pr,
mutAges,
mutPops,
mutFreqs,
Ddata='../Data/Ilya_Data/sim/sfs/working/pardis2',
suffix='',
inputParamsFiles=None,
scen2alternateNeutralParams={}):
"""Create simulation parameters for cross( mutAges, mutPops, mutFreqs ).
"""
dbg('"YYYYYYYYYYYYY" inputParamsFiles')
if inputParamsFiles == None:
inputParamsFiles = [
Ddata + '/simParams' + befAft + suffix + '.txt'
for befAft in 'Bef', 'Aft'
]
dbg('"ZZZZZZZZZZZZZ" inputParamsFiles')
pr.addInvokeRule(invokeFn=CreateSimsParams_neutral,
invokeArgs=Dict('Ddata suffix inputParamsFiles'))
for mutAge in mutAges:
for mutPop in mutPops:
for mutFreq in mutFreqs:
pr.addInvokeRule(
invokeFn=CreateSimsParams_selection,
invokeArgs=dict(
mutAges=(mutAge, ),
mutPops=(mutPop, ),
mutFreqs=(mutFreq, ),
**Dict('Ddata suffix scen2alternateNeutralParams')),
mediumRuleNameSfx=(mutAge, mutPop, mutFreq),
attrs=Dict('mutAge mutPop mutFreq'))
def evalSpatialLoc( Ddata, thinSfx, scenario, putativeMutPop, nreplicas, complikeSfx = '',
likesTableSfx = '', selpos = 500000, whichSpatialLoc = 'Spline',
getio = None ):
"""Evaluate spatial localization. Compute relevant replica statistic. For each replica,
compute: whether the localized intervals include the causal SNP; statistics about the
localized intervals; the position of the causal SNP relative to the localized intervals."""
assert not scenario.isNeutral()
snpStatsDir = os.path.join( Ddata, 'snpStats'+ thinSfx, scenario.scenDir() )
replicaStatsDir = os.path.join( Ddata, 'replicastats'+ thinSfx, scenario.scenDir() )
if putativeMutPop == None: putativeMutPop = scenario.mutPop
sfxs = ( putativeMutPop, complikeSfx, likesTableSfx )
intervalsListFN = os.path.join( replicaStatsDir, AddFileSfx( 'intervals%sList.tsv' % whichSpatialLoc, *sfxs ) )
causalGdPosFN = os.path.join( replicaStatsDir, 'causalGdPos.tsv' )
spatialLocEvalFN = os.path.join( replicaStatsDir, AddFileSfx( 'spatialLocEval%s.tsv' % whichSpatialLoc, *sfxs ) )
if getio: return dict( depends_on = ( intervalsListFN, causalGdPosFN ),
creates = spatialLocEvalFN,
mediumRuleNameSfx = scenario.scenDir(),
name = 'evalSpatialLoc_' + whichSpatialLoc )
with IDotData.openForWrite( spatialLocEvalFN,
'replicaNum numIntervals totLenBp totLenGd causalIncluded '
'distanceToIntervalBoundaryBp distanceToIntervalBoundaryGd' ) as spatialLocEvalFile:
for ( replicaNum2, replicaIntervals ), ( replicaNum3, causalGdPos, replicaNum4 ) in \
itertools.izip( IDotData( intervalsListFN ).groupby( 'replicaNum' ),
IDotData.merge( iDotDatas = ( IDotData( causalGdPosFN ),
IDotData( intervalsListFN ).replicaNum.removeDups() ),
cols = ( 'replicaNum', 'replicaNum' ) ) ):
replicaNum2, replicaNum3, replicaNum4 = map( int, ( replicaNum2, replicaNum3, replicaNum4 ) )
if not replicaNum2 == replicaNum3 == replicaNum4:
dbg( 'replicaNum2 replicaNum3 replicaNum4 intervalsListFN complikeFN causalGdPosFN spatialLocEvalFN' )
assert replicaNum2 == replicaNum3 == replicaNum4
causalIncluded = False
totLenBp = 0
totLenGd = 0.0
for replicaInterval in replicaIntervals:
dbg( 'replicaInterval causalGdPos' )
#assert bool( replicaInterval.bpFrom <= selpos <= replicaInterval.bpTo ) == bool( replicaInterval.gdFrom <= causalPos_gd <= replicaInterval.gdTo )
assert ( replicaInterval.gdFrom <= causalGdPos <= replicaInterval.gdTo ) == ( replicaInterval.bpFrom <= selpos <= replicaInterval.bpTo )
if replicaInterval.gdFrom <= causalGdPos <= replicaInterval.gdTo:
causalIncluded = True
totLenGd += ( replicaInterval.gdTo - replicaInterval.gdFrom )
totLenBp += ( replicaInterval.bpTo - replicaInterval.bpFrom )
spatialLocEvalFile.writeRecord( replicaNum2, len( replicaIntervals ),
totLenBp, totLenGd, int( causalIncluded ),
np.min( ( np.min( np.abs( replicaIntervals.bpFrom - selpos ) ),
np.min( np.abs( replicaIntervals.bpTo - selpos ) ) ) ),
np.min( ( np.min( np.abs( replicaIntervals.gdFrom - causalGdPos ) ),
np.min( np.abs( replicaIntervals.gdTo - causalGdPos ) ) ) ) )
def withoutNans( self, cols = None ): """Return a new DotData obtained from this one by removing rows that have nan values in any of the given columns""" if cols == None: cols = self.dtype.names dbg( '#zip( cols, map( type, [ numpy.isnan( self[ c ] ) for c in cols ] ) )' ) return self[ numpy.invert( reduce( operator.or_, [ numpy.isnan( self[ c ] ) for c in cols ] ) ) ]
def gather_iHS_scores( selPop, chrom, ihsFN, pop2ancFreqFN, ihsOutFN, dihhOutFN, getio = None ):
"""Gather iHS scores"""
if getio: return dict( depends_on = ( ihsFN, pop2ancFreqFN ), creates = ( ihsOutFN, dihhOutFN ),
attrs = Dict( 'chrom', pop = selPop, piperun_short = True ) )
d0 = pd.read_csv( ihsFN, sep = '\t',
usecols = ( 'Population', 'Chrom' ), nrows = 1 )
dbg( 'd0' )
assert str( d0.loc[ 0, 'Chrom' ] ) == str( chrom )
assert d0.loc[ 0, 'Population'] == selPop
d = pd.read_csv( ihsFN, sep = '\t',
usecols = ( 'SNP pos (bases)', 'Ancestral Freq', 'Both iHS', 'Both iHH_D', 'Both iHH_A' ),
index_col = 'SNP pos (bases)',
na_values = ( '-', ) )
d.index.name = 'pos'
pop2ancFreq = pd.read_table( pop2ancFreqFN, index_col = 'pos', usecols = ( 'pos', selPop, ) )
# snp2ancFreq = pd.read_table( snpInfoFN, index_col = 'SNP pos (bases)',
# usecols = ( 'SNP pos (bases)', 'Ancestral Freq' ) )
# snp2ancFreq.dropna( inplace = True )
# dbg( 'len(pop2ancFreq) len(snp2ancFreq) pop2ancFreq.index.difference(snp2ancFreq.index)' )
# dbg( 'len(pop2ancFreq) len(snp2ancFreq) snp2ancFreq.index.difference(pop2ancFreq.index)' )
# dbg( 'np.all(pop2ancFreq.index.values==snp2ancFreq.index.values)' )
# dbg( 'np.sum(pop2ancFreq.index.values==snp2ancFreq.index.values)' )
# dbg( 'len(pop2ancFreq.index.values) len(snp2ancFreq.index.values)' )
# pop2ancFreq.index.name = 'pos'
# dbg( '3 pop2ancFreq selPop pop2ancFreq.columns' )
pop2ancFreq.rename( columns = { selPop : selPop + '_ancFreq' }, inplace = True )
# dbg( '4 pop2ancFreq' )
# print "ii:", pop2ancFreq.info()
# pop2ancFreq.to_csv( 'pf.tsv', sep = '\t', header = True, na_rep = 'NaN' )
# dbg( '1 d' )
d = d.join( pop2ancFreq, how = 'right', sort = True )
# dbg( '2 d' )
# af1 = d['Ancestral Freq']
af2 = d[selPop + '_ancFreq']
# dbg( '"GGGGGGGGGG" (af1-af2).max() (af1.isnull()==af2.isnull()).all()' )
d_iHS = pd.DataFrame( data = dict( iHS = d[ 'Both iHS' ] ) )
d_iHS.to_csv( ihsOutFN, sep = '\t', header = True, na_rep = 'NaN' )
# dihh = subs.normalizeByFreq( rawVals = ( d[ 'Both iHH_D' ] - d[ 'Both iHH_A' ] ).values,
# ancfreq = 1.0 - af2.values )
d_iHH = pd.DataFrame( data = dict( iHHDiff = d[ 'Both iHH_D' ] - d[ 'Both iHH_A' ],
normingFreqs = 1.0 - af2 ) )
d_iHH.to_csv( dihhOutFN, sep = '\t', header = True, na_rep = 'NaN' )
def withoutNans(self, cols=None):
"""Return a new DotData obtained from this one by removing rows that have nan values
in any of the given columns"""
if cols == None: cols = self.dtype.names
dbg('#zip( cols, map( type, [ numpy.isnan( self[ c ] ) for c in cols ] ) )'
)
return self[numpy.invert(
reduce(operator.or_, [numpy.isnan(self[c]) for c in cols]))]
def computeMeanFstAndFreqDiffScores( pops, chrom, selPop, sweepDir,
pop2ancFreqFN, pop2sampleSizeFN, outMeanFstFN, outFreqDiffFN, getio = None ):
"""Compute meanFst and freqDiff scores"""
if selPop not in pops: pops = tuple( MakeSeq( pops ) ) + ( selPop, )
cmpPops = [ pop for pop in pops if pop != selPop ]
if getio: return dict( depends_on = ( pop2ancFreqFN, pop2sampleSizeFN ),
creates = ( outMeanFstFN, outFreqDiffFN ),
attrs = Dict( 'chrom', pop = pops ) )
# pop2ancFreq.to_csv( 'befdrop.tsv', sep = '\t' )
# pop2ancFreq.fillna( value = 1.0, inplace = True )
# pop2ancFreq.to_csv( 'aftdrop.tsv', sep = '\t' )
pop2ancFreq = pd.read_table( pop2ancFreqFN, index_col = 'pos' )
pop2sampleSize = pd.read_table( pop2sampleSizeFN, index_col = 'pop' ).sampleSize
dbg( 'pop2sampleSize' )
#pop2snpInfo.to_csv( 'test.tsv', sep = '\t', header = True )
derFreq = 1.0 - pop2ancFreq[ selPop ]
cmpAncFreqs = pop2ancFreq[ [ pop for pop in pops if pop != selPop ] ]
meanAnc = cmpAncFreqs.mean( axis = 1 )
freqDiff = derFreq - ( 1.0 - meanAnc )
freqDiff.name = 'freqDiff'
freqDiff.to_csv( outFreqDiffFN, sep = '\t', header = True )
# compute meanFst
# dbg( '"vvvvvvvvvvvw" selPop pop2ancFreq[selPop] pop2ancFreq["JPT+CHB"] pop2ancFreq["YRI"]' )
# dbg( 'selPop pop2sampleSize[selPop] pop2sampleSize["JPT+CHB"] pop2sampleSize["YRI"]' )
d = dict([ ( pop, fst_onePopPair( ancFreqs = np.array( ( pop2ancFreq[ selPop ], pop2ancFreq[ pop ] ) ),
sampleSizes = ( pop2sampleSize[ selPop ], pop2sampleSize[ pop ] ) ) )
for pop in cmpPops ])
fstVals = pd.DataFrame( data = d, index = pop2ancFreq.index )
# spc = fst_onePopPair( ancFreqs = np.array( ( pop2ancFreq[ 'BEB' ], pop2ancFreq[ 'ASN' ] ) ),
# sampleSizes = ( pop2sampleSize[ 'BEB' ], pop2sampleSize[ 'ASN' ] ) )
# dbg( '"ddddddddddd" fstVals.loc[526736] spc' )
# dbg( 'fstVals' )
fstVals.fillna( value = 0.0, inplace = True )
#fstVals.to_csv( 'fstvals.tsv', sep = '\t', header = True, na_rep = 'NaN' )
fstMean = fstVals.mean( axis = 1 )
dbg( 'fstVals fstMean' )
fstMean.name = 'meanFst'
fstMean.to_csv( outMeanFstFN, sep = '\t', header = True, na_rep = 'NaN' )
def checkTableKey(inFN,
cols,
comparison='lt',
writeCheckedFile=True,
tsvOpts={},
lineFilter=None,
lineFilterCols=(),
getio=None):
"""Check that in the given table, record identifiers increase uniformly.
Params:
cols - the columns whose tuple should uniformly inrease
comparison - this comparison must be true between each record and the next.
the comparison is the name of a routine in the operator module.
"""
cols = tuple(MakeSeq(cols))
lineFilterCols = tuple(MakeSeq(lineFilterCols))
checkedFN = Str('$inFN.checked_${comparison}') + Sfx(*cols)
if getio:
return dict(depends_on=inFN,
creates=checkedFN if writeCheckedFile else (),
attrs=dict(piperun_short=True))
comparisonFunc = getattr(operator, comparison)
prevRec = None
loadCols = cols + lineFilterCols
nskipped = 0
nchecked = 0
for i, r in enumerate(IDotData(inFN, ToLoad=loadCols, **tsvOpts)):
if lineFilter and not lineFilter(r):
nskipped += 1
continue
thisRec = r[cols] if IsSeq(r) else (r, )
if i > 0 and not comparisonFunc(prevRec, thisRec):
logging.error(
Str('at line $i of $inFN, looking at $cols: $prevRec is not $comparison $thisRec'
))
assert False
else:
nchecked += 1
prevRec = thisRec
dbg('nchecked nskipped')
DumpFile(checkedFN, 'checked ok.')
def DefineRulesTo_RunSimsAndSweep(pr,
Ddata,
simsOut='simsOut',
mutAges=AllAges,
mutPops=AllPops,
mutFreqs=AllFreqs,
nreplicas=100,
suffix='',
thinning='',
thinExt='',
tests=('lrh', 'ihs', 'xpop'),
doRunSims=True,
doRunThinning=True,
doRunSweep=True,
inputParamsFiles=[],
acceptExistingSimConfigFiles=False,
pop2name=pop2name,
runImportsLocally=True,
doOnlyStages=None,
DdataSeeds='',
setOptions=(),
appendOptions=(),
scen2alternateNeutralParams={},
useGenMap=None,
powerSfx='',
withGeneConvBug=False,
withNewCosi=False,
withCosi=None,
DdataMimic=None):
"""Define rules for running simulations and doing Sweep analyses of them.
Parameters:
mutAges, mutPops, mutFreqs - parameters defining the selection scenario.
"""
dbg('"Running_doRunSims"')
if doRunSims:
DefineRulesTo_RunSimsOnly(**Dict(
'pr mutAges mutPops mutFreqs nreplicas Ddata simsOut suffix '
'inputParamsFiles DdataSeeds useGenMap scen2alternateNeutralParams '
'withGeneConvBug withNewCosi withCosi DdataMimic'))
dbg('"Running_doThinning"')
if doRunThinning:
DefineRulesTo_DoThinning(**Dict(
'pr mutAges mutPops mutFreqs nreplicas Ddata simsOut thinning '
'thinExt suffix'))
dbg('"Running_doRunSweep"')
if doRunSweep:
DefineRulesTo_RunSweepOnSims(**Dict(
'pr mutAges mutPops mutFreqs nreplicas Ddata simsOut thinning '
'suffix tests inputParamsFiles thinExt setOptions appendOptions '
'acceptExistingSimConfigFiles pop2name runImportsLocally '
'doOnlyStages powerSfx'))
dbg('"FINISHED_runSimsAndSweep"')
def normalizeInBins_tsv( inDataFN, valCol, binCol, binMin, binMax, binStep, binsFN, outFN,
normedCol,
getio = None):
"""Normalize data within bins, using previously computed bin means"""
if getio: return dict( depends_on = ( inDataFN, binsFN ), creates = outFN, uses = normalizeInBins )
inData = pd.read_table( inDataFN )
binStats = pd.read_table( binsFN )
binMeans = binStats.means
totCount = float( binStats.counts.sum() )
totMean = binStats.sums.sum() / totCount
commonStd = np.sqrt( binStats.sumsSq.sum() / totCount - totMean * totMean )
dbg( '"CCCCCCCC" commonStd binMeans totCount totMean binStats.sums.sum() binStats.sumsSq.sum()' )
normed = normalizeInBins( **Dict( 'inData valCol binCol binMin binMax binStep binMeans commonStd' ) )
inData.insert( len( inData.columns ), normedCol, normed )
inData.to_csv( outFN, sep = '\t', na_rep = 'NaN', index = False )
def runCmdParallelized( commands, depends_on, creates, comment,
splitFunc, joinFunc, saveOutputTo = None, splitFN = None, joinFN = None,
name = None, mediumRuleName = None, getio = None ):
"""Run the specified command, using parallelization."""
from Operations.Ilya_Operations.PipeRun.python.PipeRun import PipeRun
dbg( '"IN_RUNCMDPAR_EEEEEE" depends_on creates saveOutputTo' )
if creates is None: creates = ()
commands = MakeSeq( commands )
depends_on = MakeSeq( depends_on )
creates = MakeSeq( creates )
gio = Dict( 'depends_on creates comment name mediumRuleName' )
dbg( 'gio' )
if getio: return Dict( 'depends_on creates comment name mediumRuleName saveOutputTo',
uses = ( splitFunc, joinFunc ) )
splitFN = splitFN or list( MakeSeq( depends_on ) )[0]
joinFN = RandomString(12) if saveOutputTo else ( joinFN or list( MakeSeq( creates ) )[0] )
assert any([ splitFN in command for command in commands ])
assert saveOutputTo or any([ joinFN in command for command in commands ])
logging.info( 'calling ' + str( splitFunc ) + ' to split ' + splitFN )
outDir = os.path.join( '/broad/hptmp', getpass.getuser(), 'par', os.path.abspath( splitFN )[1:] )
pr = PipeRun( name = 'splitting', descr = 'splitting' )
r = pr.addInvokeRule( invokeFn = doSplit, invokeArgs = Dict( 'splitFunc splitFN outDir' ) )
pr.runSubPipeline()
chunkFNs = SlurpFileLines( r.creates[0] )
logging.info( 'finished running ' + str( splitFunc ) + ' to split ' + splitFN )
dbg( '"CHUNKS_ARE" chunkFNs' )
pr = PipeRun( name = 'parallelizing', descr = 'parallelizing' )
chunkOutFNs = []
for chunkFN in chunkFNs:
chunkOutFN = AddFileSfx( chunkFN, 'out' )
chunkOutFNs.append( chunkOutFN )
for command in commands:
dbg( 'splitFN chunkFN chunkOutFN command command.replace(splitFN,chunkFN)' )
pr.addRule( commands = [ command.replace( splitFN, chunkFN ).replace( joinFN, chunkOutFN ) for command in commands ],
depends_on = [ f if f != splitFN else chunkFN for f in depends_on ],
creates = [ f if f != joinFN else chunkOutFN for f in creates ],
saveOutputTo = None if saveOutputTo is None else chunkOutFN )
pr.runSubPipeline()
joinFunc( inFNs = chunkOutFNs, outFN = None if saveOutputTo else joinFN )
def normalizeByFreq_compute_normed(rawVals,ancfreq, StdDev, expectation):
Frequency = np.arange(0.05, 1.05, 0.05)
#print Frequency
der_freq = 1 - ancfreq
normVal = np.repeat( np.nan, len( rawVals ) )
# Bookkeeping
dbg( 'StdDev' )
dbg( 'der_freq' )
for i in range(len(Frequency)):
idx = ((Frequency[i] - der_freq) < .05) & ( (Frequency[i] - der_freq) >= 0 ) & np.isfinite( rawVals )
normVal[ idx ] = (rawVals[ idx ] - expectation[i])/StdDev
# dbg( '"KKKKK" i Frequency[i] expectation[i] idx.nonzero() rawVals[idx] normVal[idx]' )
return normVal
def joinStats( snpInfoFN, statLikesFNs, likesRatioFN, outFN, getio = None ):
"""Join stats into one file"""
if getio:
return dict( depends_on = ( snpInfoFN, likesRatioFN ) + tuple( MakeSeq( statLikesFNs ) ),
creates = outFN )
snpInfo = pd.read_table( snpInfoFN, index_col = 'SNP pos (bases)' )
snpInfo.index.rename( 'pos', inplace = True )
statLikes = [ pd.read_table( statLikeFN, index_col = 'pos' ) for statLikeFN in statLikesFNs ]
likesRatio = pd.read_table( likesRatioFN, index_col = 'pos' )
result = snpInfo.join( statLikes + [ likesRatio ], how = 'outer' )
result.info()
dbg( 'result.describe()' )
result.to_csv( outFN, sep = '\t', na_rep = 'NaN', header = True )
def StartPushers( use_args = None, as_daemons = False ):
parser = optparse.OptionParser()
parser.add_option( '-P', '--num-result-pushers', type='int', dest = 'numResultPushers',
help='create NUMPROCS parallel result pushers', metavar='NUMPROCS',
default=1 )
parser.add_option( '-Q', '--queues', dest = 'queues',
default = os.path.join( '..', 'Other', 'queues', getpass.getuser() ),
help='push results for the specified QUEUES', metavar='QUEUES' )
parser.add_option( '-S', '--sleepInterval', type='int',
help = 'between checks, sleep for SEC seconds', metavar = 'SEC', default = 20 )
dbg( 'use_args' )
(options, args) = parser.parse_args( args = sys.argv[1:] if use_args is None else list( use_args ) )
dbg( 'options args' )
assert not args
for i in range( options.numResultPushers ):
p = multiprocessing.Process( target = PushResults, args = ( options, ) )
allPushers.append( p )
p.daemon = as_daemons
p.start()
time.sleep( min( 1.0, random.normalvariate( 4.0, 1.0 ) ) )
def normalizeByFreq_compute_normed(rawVals, ancfreq, StdDev, expectation):
Frequency = np.arange(0.05, 1.05, 0.05)
#print Frequency
der_freq = 1 - ancfreq
normVal = np.repeat(np.nan, len(rawVals))
# Bookkeeping
dbg('StdDev')
dbg('der_freq')
for i in range(len(Frequency)):
idx = ((Frequency[i] - der_freq) < .05) & (
(Frequency[i] - der_freq) >= 0) & np.isfinite(rawVals)
normVal[idx] = (rawVals[idx] - expectation[i]) / StdDev
# dbg( '"KKKKK" i Frequency[i] expectation[i] idx.nonzero() rawVals[idx] normVal[idx]' )
return normVal
def joinStats(snpInfoFN, statLikesFNs, likesRatioFN, outFN, getio=None):
"""Join stats into one file"""
if getio:
return dict(depends_on=(snpInfoFN, likesRatioFN) +
tuple(MakeSeq(statLikesFNs)),
creates=outFN)
snpInfo = pd.read_table(snpInfoFN, index_col='SNP pos (bases)')
snpInfo.index.rename('pos', inplace=True)
statLikes = [
pd.read_table(statLikeFN, index_col='pos')
for statLikeFN in statLikesFNs
]
likesRatio = pd.read_table(likesRatioFN, index_col='pos')
result = snpInfo.join(statLikes + [likesRatio], how='outer')
result.info()
dbg('result.describe()')
result.to_csv(outFN, sep='\t', na_rep='NaN', header=True)
def DefineRulesTo_CreateSimulationParams( pr, mutAges, mutPops, mutFreqs,
Ddata = '../Data/Ilya_Data/sim/sfs/working/pardis2',
suffix = '', inputParamsFiles = None,
scen2alternateNeutralParams = {} ):
"""Create simulation parameters for cross( mutAges, mutPops, mutFreqs ).
"""
dbg( '"YYYYYYYYYYYYY" inputParamsFiles' )
if inputParamsFiles == None:
inputParamsFiles = [ Ddata + '/simParams' + befAft + suffix + '.txt' for befAft in 'Bef', 'Aft' ]
dbg( '"ZZZZZZZZZZZZZ" inputParamsFiles' )
pr.addInvokeRule( invokeFn = CreateSimsParams_neutral, invokeArgs = Dict( 'Ddata suffix inputParamsFiles' ) )
for mutAge in mutAges:
for mutPop in mutPops:
for mutFreq in mutFreqs:
pr.addInvokeRule( invokeFn = CreateSimsParams_selection,
invokeArgs = dict( mutAges = ( mutAge, ), mutPops = (mutPop,),
mutFreqs = (mutFreq,),
**Dict( 'Ddata suffix scen2alternateNeutralParams' ) ),
mediumRuleNameSfx = ( mutAge, mutPop, mutFreq ),
attrs = Dict( 'mutAge mutPop mutFreq' ) )
def LoadComparison( pop ):
"""Load comparison with one pop"""
d0 = pd.read_csv( pop2FN[ pop ], sep = '\t',
usecols = ( 'Pop 1', 'Pop 2', 'Chrom' ), nrows = 1 )
dbg( 'd0' )
assert str( d0.loc[ 0, 'Chrom' ] ) == str( chrom )
assert ( d0.loc[ 0, 'Pop 1'] == selPop and d0.loc[0, 'Pop 2'] == pop ) or ( d0.loc[0,'Pop 1'] == pop and d0.loc[0,'Pop 2'] == selPop )
flip = ( d0.loc[0,'Pop 1'] == pop )
d = pd.read_csv( pop2FN[ pop ], sep = '\t',
usecols = ( 'SNP pos (bases)', 'L AllEHH logratio Deviation', 'R AllEHH logratio Deviation' ),
index_col = 'SNP pos (bases)',
na_values = ( '-', ) )
d.info()
if flip:
d[ 'L AllEHH logratio Deviation' ] *= -1
d[ 'R AllEHH logratio Deviation' ] *= -1
return pd.DataFrame.from_dict({ pop : d.max( axis = 1 ) })
def checkTableKey( inFN, cols, comparison = 'lt', writeCheckedFile = True,
tsvOpts = {}, lineFilter = None, lineFilterCols = (), getio = None ):
"""Check that in the given table, record identifiers increase uniformly.
Params:
cols - the columns whose tuple should uniformly inrease
comparison - this comparison must be true between each record and the next.
the comparison is the name of a routine in the operator module.
"""
cols = tuple( MakeSeq( cols ) )
lineFilterCols = tuple( MakeSeq( lineFilterCols ) )
checkedFN = Str( '$inFN.checked_${comparison}' ) + Sfx( *cols )
if getio: return dict( depends_on = inFN, creates = checkedFN if writeCheckedFile else (),
attrs = dict( piperun_short = True ) )
comparisonFunc = getattr( operator, comparison )
prevRec = None
loadCols = cols + lineFilterCols
nskipped = 0
nchecked = 0
for i, r in enumerate( IDotData( inFN, ToLoad = loadCols, **tsvOpts ) ):
if lineFilter and not lineFilter( r ):
nskipped += 1
continue
thisRec = r[ cols ] if IsSeq( r ) else ( r, )
if i > 0 and not comparisonFunc( prevRec, thisRec ):
logging.error( Str( 'at line $i of $inFN, looking at $cols: $prevRec is not $comparison $thisRec' ) )
assert False
else: nchecked += 1
prevRec = thisRec
dbg( 'nchecked nskipped' )
DumpFile( checkedFN, 'checked ok.' )
def computeLikeRatioForStat_do( statVals, hitLikes, missLikes, bins ):
"""Compute likes ratio"""
# Precompute the likelihood ratio corresponding to each bin
dbg( 'statVals hitLikes missLikes bins' )
indNaN = hitLikes != 1e-10
missingVal = np.log( np.min( hitLikes[indNaN] / missLikes[indNaN] ) )
CLR = [ ( np.log( hitLike / missLike ) if hitLike != 1e-10 else missingVal ) if hitLike != 0.0 else np.nan
for hitLike, missLike in zip( hitLikes, missLikes ) ]
CLR = np.array( [ CLR[0] ] + CLR + [ CLR[-1] ] )
binIds = np.digitize( statVals.values, bins )
st_binSize = ( bins[1] - bins[0] )
st_nbins = len( bins ) -1
binIds2 = np.where( np.isfinite( statVals.values ),
np.clip( ( ( statVals.values - bins[0] ) / st_binSize ).astype( np.int16 ), 0, st_nbins-1 ),
len( hitLikes ) ) + 1
return np.where( np.isnan( statVals.values ),
np.repeat( np.nan, len( statVals ) ), CLR[ binIds ] ), binIds, binIds2
def computeMeanStd( inFNs, colName, outFN, getio = None ):
"""Compute mean and std using blaze"""
if getio: return dict( depends_on = inFNs, creates = outFN )
filenames = inFNs
dbg( 'inFNs' )
sk = StatKeeper()
for f in filenames:
dbg( 'f' )
d = pd.read_table( f )
dbg( 'f len(d)' )
sk.addVals( d[ colName ].values )
pd.DataFrame( dict( stat = 'mean std count numNaNs'.split(),
val = ( sk.getMean(), sk.getStd(), sk.getCount(), sk.getNumNaNs() ) ) ).to_csv( outFN,
sep = '\t',
index = False,
na_rep = 'NaN' )
def computeMeanStd(inFNs, colName, outFN, getio=None):
"""Compute mean and std using blaze"""
if getio: return dict(depends_on=inFNs, creates=outFN)
filenames = inFNs
dbg('inFNs')
sk = StatKeeper()
for f in filenames:
dbg('f')
d = pd.read_table(f)
dbg('f len(d)')
sk.addVals(d[colName].values)
pd.DataFrame(
dict(stat='mean std count numNaNs'.split(),
val=(sk.getMean(), sk.getStd(), sk.getCount(),
sk.getNumNaNs()))).to_csv(outFN,
sep='\t',
index=False,
na_rep='NaN')
def gather_iHS_scores(selPop,
chrom,
ihsFN,
pop2ancFreqFN,
ihsOutFN,
dihhOutFN,
getio=None):
"""Gather iHS scores"""
if getio:
return dict(depends_on=(ihsFN, pop2ancFreqFN),
creates=(ihsOutFN, dihhOutFN),
attrs=Dict('chrom', pop=selPop, piperun_short=True))
d0 = pd.read_csv(ihsFN, sep='\t', usecols=('Population', 'Chrom'), nrows=1)
dbg('d0')
assert str(d0.loc[0, 'Chrom']) == str(chrom)
assert d0.loc[0, 'Population'] == selPop
d = pd.read_csv(ihsFN,
sep='\t',
usecols=('SNP pos (bases)', 'Ancestral Freq', 'Both iHS',
'Both iHH_D', 'Both iHH_A'),
index_col='SNP pos (bases)',
na_values=('-', ))
d.index.name = 'pos'
pop2ancFreq = pd.read_table(pop2ancFreqFN,
index_col='pos',
usecols=(
'pos',
selPop,
))
# snp2ancFreq = pd.read_table( snpInfoFN, index_col = 'SNP pos (bases)',
# usecols = ( 'SNP pos (bases)', 'Ancestral Freq' ) )
# snp2ancFreq.dropna( inplace = True )
# dbg( 'len(pop2ancFreq) len(snp2ancFreq) pop2ancFreq.index.difference(snp2ancFreq.index)' )
# dbg( 'len(pop2ancFreq) len(snp2ancFreq) snp2ancFreq.index.difference(pop2ancFreq.index)' )
# dbg( 'np.all(pop2ancFreq.index.values==snp2ancFreq.index.values)' )
# dbg( 'np.sum(pop2ancFreq.index.values==snp2ancFreq.index.values)' )
# dbg( 'len(pop2ancFreq.index.values) len(snp2ancFreq.index.values)' )
# pop2ancFreq.index.name = 'pos'
# dbg( '3 pop2ancFreq selPop pop2ancFreq.columns' )
pop2ancFreq.rename(columns={selPop: selPop + '_ancFreq'}, inplace=True)
# dbg( '4 pop2ancFreq' )
# print "ii:", pop2ancFreq.info()
# pop2ancFreq.to_csv( 'pf.tsv', sep = '\t', header = True, na_rep = 'NaN' )
# dbg( '1 d' )
d = d.join(pop2ancFreq, how='right', sort=True)
# dbg( '2 d' )
# af1 = d['Ancestral Freq']
af2 = d[selPop + '_ancFreq']
# dbg( '"GGGGGGGGGG" (af1-af2).max() (af1.isnull()==af2.isnull()).all()' )
d_iHS = pd.DataFrame(data=dict(iHS=d['Both iHS']))
d_iHS.to_csv(ihsOutFN, sep='\t', header=True, na_rep='NaN')
# dihh = subs.normalizeByFreq( rawVals = ( d[ 'Both iHH_D' ] - d[ 'Both iHH_A' ] ).values,
# ancfreq = 1.0 - af2.values )
d_iHH = pd.DataFrame(data=dict(iHHDiff=d['Both iHH_D'] - d['Both iHH_A'],
normingFreqs=1.0 - af2))
d_iHH.to_csv(dihhOutFN, sep='\t', header=True, na_rep='NaN')
def mergeSims( scenario, Ddata, posFileFN = None, simsOut = 'simsOut', nreplicas = 100, thinExt = '', thinSfx = '',
putativeMutPop = None, outFile = None,
pop2name = pop2name, statsSfx = '', ihsSfx = '',
limitToPop = None,
getio = None ):
"""Gathers per-SNP information, for all replicas of a given scenario, and outputs it in a single DotData where each line
gives info for one SNP.
Specifically, reads simulation and Sweep output, collects columns needed for composite likehood test (chrom, base pair position, genetic
distance, anc frequencies for 3 populations, xpop for each pair, and ihs, iHH_A and iHH_D for selected population)
Input params:
scenario - an object of class Scenario, indicating the simulation scenario (either neutral or a selection scenario)
from which all replicas were simulated.
nreplicas - the number of replicas simulated under this scenario.
Each replica represents a chromosome region, with a set of SNPs on it.
Ddata - the directory under which the simulations and the Sweep analysis results live.
Under this directory we expect to find:
iHS analysis results, under power_ihs/
XP-EHH analysis results, under power_xpop
simulation output giving SNP positions
thinExt - the extension appended to simulation files that describe the SNPs in the simulated replica.
Sometimes we create simulations and then thin them under different thinning models (to simulate SNP ascertainment
by the various stages of HapMap; these differently thinned versions of the same simulations might be stored in
simulation files with different extensions.
thinSfx - the suffix appended to the power_ihs and power_xpop directory names, telling where to find iHS and XP-EHH
analyses of the simulations. When we analyze the same simulations after applying different thinning scenarios,
the iHS and XP-EHH analyses for each thinning scenario go into a separate set of directories.
putativeMutPop - the population in which, we think, selection is occurring, aka "putatively selected population".
In practice, when localizing a given region, we will usually suspect that selection has occurred in a particular
population. When doing a genome-wide scan, we can do several scans assuming each population in turn to be
the selected population, and find regions selected in that population.
Output params:
Ddata - under Ddata writes a DotData named merged_scenName.data, where each line gives info
for one SNP, with the following columns (type of data is float unless stated otherwise):
CHROM_POS 1 - physical (basepair) position of the SNP within its replica.
Note that one merged file contains SNPs for a set of replicas (all for the same scenario),
so there could be multiple SNPs with the same position. The replica number
is given in the Chrom column.
FREQ1 1 - derived allele frequency in pop 1 ( European )
FREQ1 4 - derived allele frequency in pop 4 ( EastAsian )
FREQ1 5 - derived allele frequency in pop 5 ( WestAfrican )
R AllEHH logratio Deviation European_WestAfrican - XP-EHH score to the right of the SNP,
between European and WestAfrican pops, normalized to the neutral background.
Analogously for the next five columns:
L AllEHH logratio Deviation European_WestAfrican
R AllEHH logratio Deviation EastAsian_European
L AllEHH logratio Deviation EastAsian_European
R AllEHH logratio Deviation EastAsian_WestAfrican
L AllEHH logratio Deviation EastAsian_WestAfrican
SNP pos (cM) European_WestAfrican - genetic map position of this SNP, within its replica.
(the European_WestAfrican suffix is irrelevant).
SNP pos (bases) European_WestAfrican - physical (basepair) position of this SNP within its replica.
(the European_WestAfrican suffix is irrelevant).
Chrom European_WestAfrican - the replica from which this SNP comes; can be nan.
(the European_WestAfrican suffix is irrelevant)
Chrom - the replica from which this SNP comes; can be nan
SNP pos (bases) - physical (basepair) position of this SNP within its replica.
SNP pos (cM) - genetic map position of this SNP within its replica
Both iHH_A - sum of iHH_A for both directions from this SNP
Both iHH_D - sum of iHH_D for both directions from this SNP
Both iHS - the value in 'Both Unstandardised iHS' (below), but binned by derived allele frequency
and normalized within the bin.
Left iHH_D - iHH_D to the left of the SNP (the raw integral value). analogously for the next three.
Right iHH_D
Left iHH_A
Right iHH_A
Both Unstandardised iHS - log( (iHH_A_left + iHH_A_right) / ( iHH_D_left + iHH_D_right ) )
( see also 'Both iHS' column for the standardized iHS score )
"""
if not Ddata.endswith('/'): Ddata += '/'
assert nreplicas > 0
dbg( 'pop2name' )
SimDir = os.path.join( Ddata, simsOut + thinSfx )
scenName = scenario.scenName()
scenDir = scenario.scenDir()
if putativeMutPop == None: putativeMutPop = scenario.mutPop
ihsSignifFN = os.path.join( Ddata, 'power_ihs' + thinSfx, scenDir,
'ihs_sig_' + pop2name[ putativeMutPop ] + ihsSfx + '.tsv' )
popNames = sorted( pop2name.values() )
popNums = sorted( pop2name.keys() )
minPopNum = popNums[ 0 ]
posFileKeyCols = ( 'replicaNum', 'CHROM_POS %d' % minPopNum )
xpopIhsKeyCols = ('Chrom', 'SNP pos (bases)')
popPairs = [ '%s_%s' % ( popNames[ pop1idx ], popNames[ pop2idx ] )
for pop1idx in range( len( popNames ) ) for pop2idx in range( pop1idx+1, len( popNames ) )
if limitToPop is None or limitToPop in ( popNames[ pop1idx ], popNames[ pop2idx ] ) ]
xpopSignifFNs = [ os.path.join( Ddata, 'power_xpop' + thinSfx, scenDir, 'xpop_significance_' + popPair + '.tsv' )
for popPair in popPairs ]
snpStatsDir = os.path.join( Ddata, 'snpStats' + thinSfx, scenario.scenDir() )
mergedData = outFile if outFile else os.path.join( snpStatsDir, AddFileSfx( 'merged.tsv', statsSfx, putativeMutPop, ihsSfx ) )
fileDescrs = \
{ mergedData :
( 'Various per-snp statistics for SNPs in scenario $scenario, replicas 0-$nreplicas, '
'assuming selection in ' + pop2name[ putativeMutPop ],
( ( 'CHROM_POS 1', 'physical (basepair) position of the SNP within its replica. '
'Note that one merged file contains SNPs for a set of replicas (all for the same scenario), '
'so there could be multiple SNPs with the same position. The replica number '
'is given in the Chrom column. ' ),
( 'FREQ1 1', 'derived allele frequency in pop 1 ( European )' ),
( 'R AllEHH logratio Deviation European_WestAfrican', 'XP-EHH score to the R of the SNP, '
'between European and WestAfrican pops, normalized to the neutral background.' ),
( 'SNP pos (cM) European_WestAfrican', 'genetic map SNP position' ),
( 'SNP pos (bases) European_WestAfrican', 'physical SNP position' ),
( 'Chrom European_WestAfrican', 'chromosome (or replica number)' ),
( 'Chrom', 'chromosome (or replica number)' ),
( 'SNP pos (bases)', 'physical SNP position' ),
( 'SNP pos (cM)', 'genetic map SNP position' ),
( 'Both iHH_A', 'sum of iHH_A scores for both sides' ),
( 'Both iHH_D', 'sum of iHH_D scores for both sides' ),
( 'Both iHS', 'sum of iHS scores for both sides' ),
( ' Left iHH_D', 'iHH_D score to the left of the SNP' ),
( 'Right iHH_D', 'iHH_D score to the right of the SNP' ),
( 'Left iHH_A', 'iHH_A score to the left of the SNP' ),
( 'Right iHH_A', 'iHH_A score to the right of the SNP' ),
( 'Both Unstandardised iHS', 'sum of unstandardized iHS scores for both sides' ) ) ) }
if posFileFN is None: posFileFN = os.path.join( Ddata, 'snpStats' + thinSfx, scenario.scenDir(),
AddFileSfx( 'mergedPosStacked.tsv', statsSfx, putativeMutPop, ihsSfx ) )
if getio: return dict( depends_on = [ posFileFN, ihsSignifFN ] + xpopSignifFNs, creates = mergedData,
splitByCols = dict([ ( posFileFN, dict( keyCols = posFileKeyCols ) ) ]
+ [ ( signifFN, dict( keyCols = xpopIhsKeyCols ) ) for signifFN in [ ihsSignifFN ] + xpopSignifFNs ] ),
mediumRuleNameSfx = ( scenario.scenDir(), putativeMutPop ),
fileDescrs = fileDescrs,
attrs = Dict( 'putativeMutPop nreplicas pop2name' ) )
dashFixer = lambda v: v if v != '-' else np.nan
ihsAll = IDotData(ihsSignifFN, valueFixer = dashFixer)
ihsAll = ihsAll[('Chrom','SNP pos (bases)','SNP pos (cM)','Both iHH_A','Both iHH_D','Both iHS',
'Left iHH_D','Right iHH_D','Left iHH_A','Right iHH_A','Both Unstandardised iHS')]
def chkReplica( r, n = nreplicas ): return r.Chrom < n
ihsAll = ihsAll.takewhile( chkReplica )
xpopCols = ('Chrom','SNP pos (bases)','SNP pos (cM)','L AllEHH logratio Deviation','R AllEHH logratio Deviation')
xpopSignif = tuple( [ IDotData(xpopSignifFN, valueFixer = dashFixer)[ xpopCols].takewhile( chkReplica )
for xpopSignifFN in xpopSignifFNs ] )
posCols = ['CHROM_POS %d' % minPopNum ] + ['FREQ1 %d' % popNum for popNum in popNums]
result = IDotData.merge( iDotDatas = ( IDotData( posFileFN ), ) + xpopSignif + ( ihsAll, ),
cols = (posFileKeyCols,) +
(xpopIhsKeyCols,) * ( len( popPairs ) + 1 ),
blanks = (None,) + (np.nan,) * ( len( popPairs ) + 1 ),
suffixes = ['pos'] + [ ' %s' % popPair for popPair in popPairs ] + [ '' ] )
aPopPair = 'European_WestAfrican' if 'European_WestAfrican' in popPairs else popPairs[0]
useCols = [ 'replicaNum' ] + posCols + \
[ '%s AllEHH logratio Deviation %s' % ( side, popPair ) for popPair in popPairs for side in ( 'L', 'R' ) ] + \
[ 'SNP pos (cM) ' + aPopPair,
'SNP pos (bases) ' + aPopPair,
'Chrom ' + aPopPair,
'Chrom',
'SNP pos (bases)',
'SNP pos (cM)',
'Both iHH_A',
'Both iHH_D',
'Both iHS' ]
if len( popPairs ) == 1:
result = result.renameCols( { 'L AllEHH logratio Deviation' : 'L AllEHH logratio Deviation ' + aPopPair,
'R AllEHH logratio Deviation' : 'R AllEHH logratio Deviation ' + aPopPair } )
result[ useCols ].save( mergedData )
logging.info( 'Finished mergeSims()' )
def fst_onePopPair(ancFreqs, sampleSizes):
"""Compute fst between two pops, for each SNP, given the ancestral freq in each pop and the sample sizes.
"""
dbg( 'ancFreqs sampleSizes' )
n = sampleSizes
n_tot = n[0] + n[1]
# nanc = np.ceil([ ancFreqs[0] * sampleSizes[0], ancFreqs[1] * sampleSizes[1] ])
nanc = ( np.array([ ancFreqs[0] * sampleSizes[0], ancFreqs[1] * sampleSizes[1] ]) + .5 ).astype( int )
dbg( 'nanc' )
f1 = ancFreqs
f2 = 1 - ancFreqs
dbg( 'f1 f2 f1.shape f2.shape' )
# Use Weir-Hill estimator for Fst
pmean = (nanc[0] + nanc[1]) / n_tot
nic = n[0] - (n[0]*n[0])/n_tot
njc = n[1] - (n[1]*n[1])/n_tot
dbg( 'pmean.shape nic.shape njc.shape nic njc' )
#assert nic == njc == 60
nc = nic + njc
#print nc, f1[0], f1[1]
#print f2[0], f2[1]
msp = n[0] * (f1[0] - pmean) * (f1[0] - pmean) \
+ n[1] * (f1[1] - pmean) * (f1[1] - pmean)
msg = ((n[0] * f1[0]* f2[0]) + (n[1] * f1[1] * f2[1])) \
/ (n[0] - 1 + n[1] - 1)
dbg( 'msp.shape msg.shape' )
num = msp - msg
denom = msp + (msg*(nc - 1))
#print msp, msg
#print num, denom
dbg( 'num.shape denom.shape' )
denom [ denom == 0 ] = nan
an_fst = num / denom
dbg( 'an_fst.shape' )
ipop = 0
jpop = 1
dbg( '"KKKKKKKKKKKKK" pmean nanc[ipop] nanc[jpop] n[ipop] n[jpop] nic njc nc msp msg num denom an_fst' )
return an_fst
def SetStopSignal(sigNum, stkFrm):
logging.info('Setting stop signal to stop pushers')
stopSignal[0] = True
dbg('"aftset" stopSignal')
def runCmdParallelized(commands,
depends_on,
creates,
comment,
splitFunc,
joinFunc,
saveOutputTo=None,
splitFN=None,
joinFN=None,
name=None,
mediumRuleName=None,
getio=None):
"""Run the specified command, using parallelization."""
from Operations.Ilya_Operations.PipeRun.python.PipeRun import PipeRun
dbg('"IN_RUNCMDPAR_EEEEEE" depends_on creates saveOutputTo')
if creates is None: creates = ()
commands = MakeSeq(commands)
depends_on = MakeSeq(depends_on)
creates = MakeSeq(creates)
gio = Dict('depends_on creates comment name mediumRuleName')
dbg('gio')
if getio:
return Dict(
'depends_on creates comment name mediumRuleName saveOutputTo',
uses=(splitFunc, joinFunc))
splitFN = splitFN or list(MakeSeq(depends_on))[0]
joinFN = RandomString(12) if saveOutputTo else (
joinFN or list(MakeSeq(creates))[0])
assert any([splitFN in command for command in commands])
assert saveOutputTo or any([joinFN in command for command in commands])
logging.info('calling ' + str(splitFunc) + ' to split ' + splitFN)
outDir = os.path.join('/broad/hptmp', getpass.getuser(), 'par',
os.path.abspath(splitFN)[1:])
pr = PipeRun(name='splitting', descr='splitting')
r = pr.addInvokeRule(invokeFn=doSplit,
invokeArgs=Dict('splitFunc splitFN outDir'))
pr.runSubPipeline()
chunkFNs = SlurpFileLines(r.creates[0])
logging.info('finished running ' + str(splitFunc) + ' to split ' + splitFN)
dbg('"CHUNKS_ARE" chunkFNs')
pr = PipeRun(name='parallelizing', descr='parallelizing')
chunkOutFNs = []
for chunkFN in chunkFNs:
chunkOutFN = AddFileSfx(chunkFN, 'out')
chunkOutFNs.append(chunkOutFN)
for command in commands:
dbg('splitFN chunkFN chunkOutFN command command.replace(splitFN,chunkFN)'
)
pr.addRule(
commands=[
command.replace(splitFN, chunkFN).replace(joinFN, chunkOutFN)
for command in commands
],
depends_on=[f if f != splitFN else chunkFN for f in depends_on],
creates=[f if f != joinFN else chunkOutFN for f in creates],
saveOutputTo=None if saveOutputTo is None else chunkOutFN)
pr.runSubPipeline()
joinFunc(inFNs=chunkOutFNs, outFN=None if saveOutputTo else joinFN)
def evalSpatialLoc(Ddata,
thinSfx,
scenario,
putativeMutPop,
nreplicas,
complikeSfx='',
likesTableSfx='',
selpos=500000,
whichSpatialLoc='Spline',
getio=None):
"""Evaluate spatial localization. Compute relevant replica statistic. For each replica,
compute: whether the localized intervals include the causal SNP; statistics about the
localized intervals; the position of the causal SNP relative to the localized intervals."""
assert not scenario.isNeutral()
snpStatsDir = os.path.join(Ddata, 'snpStats' + thinSfx, scenario.scenDir())
replicaStatsDir = os.path.join(Ddata, 'replicastats' + thinSfx,
scenario.scenDir())
if putativeMutPop == None: putativeMutPop = scenario.mutPop
sfxs = (putativeMutPop, complikeSfx, likesTableSfx)
intervalsListFN = os.path.join(
replicaStatsDir,
AddFileSfx('intervals%sList.tsv' % whichSpatialLoc, *sfxs))
causalGdPosFN = os.path.join(replicaStatsDir, 'causalGdPos.tsv')
spatialLocEvalFN = os.path.join(
replicaStatsDir,
AddFileSfx('spatialLocEval%s.tsv' % whichSpatialLoc, *sfxs))
if getio:
return dict(depends_on=(intervalsListFN, causalGdPosFN),
creates=spatialLocEvalFN,
mediumRuleNameSfx=scenario.scenDir(),
name='evalSpatialLoc_' + whichSpatialLoc)
with IDotData.openForWrite(
spatialLocEvalFN,
'replicaNum numIntervals totLenBp totLenGd causalIncluded '
'distanceToIntervalBoundaryBp distanceToIntervalBoundaryGd'
) as spatialLocEvalFile:
for ( replicaNum2, replicaIntervals ), ( replicaNum3, causalGdPos, replicaNum4 ) in \
zip( IDotData( intervalsListFN ).groupby( 'replicaNum' ),
IDotData.merge( iDotDatas = ( IDotData( causalGdPosFN ),
IDotData( intervalsListFN ).replicaNum.removeDups() ),
cols = ( 'replicaNum', 'replicaNum' ) ) ):
replicaNum2, replicaNum3, replicaNum4 = list(
map(int, (replicaNum2, replicaNum3, replicaNum4)))
if not replicaNum2 == replicaNum3 == replicaNum4:
dbg('replicaNum2 replicaNum3 replicaNum4 intervalsListFN complikeFN causalGdPosFN spatialLocEvalFN'
)
assert replicaNum2 == replicaNum3 == replicaNum4
causalIncluded = False
totLenBp = 0
totLenGd = 0.0
for replicaInterval in replicaIntervals:
dbg('replicaInterval causalGdPos')
#assert bool( replicaInterval.bpFrom <= selpos <= replicaInterval.bpTo ) == bool( replicaInterval.gdFrom <= causalPos_gd <= replicaInterval.gdTo )
assert (replicaInterval.gdFrom <= causalGdPos <=
replicaInterval.gdTo) == (replicaInterval.bpFrom <=
selpos <=
replicaInterval.bpTo)
if replicaInterval.gdFrom <= causalGdPos <= replicaInterval.gdTo:
causalIncluded = True
totLenGd += (replicaInterval.gdTo - replicaInterval.gdFrom)
totLenBp += (replicaInterval.bpTo - replicaInterval.bpFrom)
spatialLocEvalFile.writeRecord(
replicaNum2, len(replicaIntervals), totLenBp, totLenGd,
int(causalIncluded),
np.min((np.min(np.abs(replicaIntervals.bpFrom - selpos)),
np.min(np.abs(replicaIntervals.bpTo - selpos)))),
np.min((np.min(np.abs(replicaIntervals.gdFrom - causalGdPos)),
np.min(np.abs(replicaIntervals.gdTo - causalGdPos)))))
def PushResults(options):
"""Push finished results"""
logging.info(
'Starting result pusher as process %d on host %s with options %s' %
(os.getpid(), GetHostName(), options))
# check that all queues exist
assert all(
os.path.exists(queue) and os.path.isdir(queue)
for queue in options.queues.split(os.pathsep))
stopSignal = [False]
def SetStopSignal(sigNum, stkFrm):
logging.info('Setting stop signal to stop pushers')
stopSignal[0] = True
dbg('"aftset" stopSignal')
signal.signal(signal.SIGUSR2, SetStopSignal)
while not stopSignal[0]:
for queue in options.queues.split(os.pathsep):
logging.info('Pushing results in ' + queue + '...')
# find an unclaimed task in this queue, and try to claim it
try:
taskDirs = [f for f in os.listdir(queue) if f.startswith('mq')]
except EnvironmentError as e:
logging.info('Error getting list of tasks in queue ' + queue +
': ' + str(e))
# sleep a bit -- maybe it's some transient condition that will resolve itself
time.sleep(60 + random.normalvariate(3.0, 1.0))
continue
for taskDir in taskDirs:
fullTaskDir = os.path.join(queue, taskDir)
try:
pushingFN = os.path.join(fullTaskDir, 'pushing.dat')
submithostFN = os.path.join(fullTaskDir, 'submithost.txt')
if os.path.exists( os.path.join( fullTaskDir, 'completed.dat' ) ) \
and os.path.exists( submithostFN ) and GetSubmitHost( submithostFN ) == GetHostName() \
and not os.path.exists( pushingFN ):
try:
fd = os.open(pushingFN,
os.O_CREAT | os.O_EXCL | os.O_WRONLY)
except EnvironmentError as e:
if e.errno not in (errno.EEXIST, errno.EACCES,
errno.EAGAIN):
raise
# another resultPusher beat us to this task -- go and check other tasks
continue
os.write(
fd,
'result being pushed by process %d on host %s' %
(os.getpid(), GetHostName()))
os.close(fd)
taskDescr = ''
try:
attrsFN = os.path.join(fullTaskDir, 'attrs.tsv')
if os.path.exists(attrsFN):
taskDescr += ' output in ' + GetTaskAttr(
attrsFN, 'piperun_outputSavedTo')
except EnvironmentError as e:
logging.info('Could not read attrs for task in ' +
fullTaskDir + ': ' + str(e))
try:
infoFNs = [
os.path.join(fullTaskDir, f)
for f in ('command.dat', 'attrs.tsv',
'claimed.dat')
]
infoContents = '\n'.join([
SlurpFile(f)
if os.path.exists(f) else 'missing file: ' + f
for f in infoFNs
])
thePipe = os.open(
os.path.join(fullTaskDir, 'getresult'),
os.O_WRONLY | os.O_NONBLOCK)
exitCodeReadOk = False
writeOk = False
closeOk = False
exitCode = 'UNKNOWN'
try:
exitCode = SlurpFile(
os.path.join(fullTaskDir,
'exitCode.dat')).strip()
exitCodeReadOk = True
os.write(thePipe, exitCode)
writeOk = True
finally:
os.close(thePipe)
closeOk = True
logging.info(
'Pushed result in ' + fullTaskDir + ': ' +
('nonzero ' if exitCode != '0' else '') +
'exit code ' + exitCode + taskDescr)
if not writeOk or not closeOk or not exitCodeReadOk:
dbg('exitCodeReadOk writeOk closeOk')
if exitCodeReadOk and exitCode != '0':
logging.info(infoContents)
except EnvironmentError as e:
logging.info('The task at ' + fullTaskDir +
' seems to have been orphaned: ' +
e.strerror)
except EnvironmentError as e:
logging.info('Error processing queue ' + queue + ' task ' +
taskDir + ': ' + str(e))
# sleep a bit -- maybe it's some transient condition that will resolve itself
time.sleep(60 + random.normalvariate(3.0, 1.0))
# if we pushed at least something, go back and try again. if not, wait.
time.sleep(options.sleepInterval + random.normalvariate(3.0, 1.0))
parser.add_argument( '--test-name', help = 'name of test; is name of directory under tests/ where test is saved' )
parser.add_argument( '--bootstrap-count', type = int, default = 1000, help = 'number of boostrap iters' )
parser.add_argument( '--cosi-binary', default = './coalescent', help = 'cosi binary to run' )
print 'calling parser'
args = parser.parse_args()
print 'parser done'
# do a reference run
print 'generating reference'
SystemSucceed( ' '.join(map( str, (args.cosi_binary, '-p', '1_simple.cosiParams', '-n', 100, '-m' ))) + ' | sample_stats_extra > ref.tsv' )
refData = DotData( SVPath = 'ref.tsv' )
min_p = np.ones( len( refData.dtype.names ) )
max_D = np.repeat( -np.inf, len( refData.dtype.names ) )
for i in range( 10 ):
dbg( 'i' )
refFN = 'reftest%d.tsv' % i
SystemSucceed( ' '.join(map( str, ( args.cosi_binary, '-p', '0_simple.cosiParams', '-n', 100, '-m' ))) + ' | sample_stats_extra > ' + refFN )
z = DotData( SVPath = refFN )
for colNum, col in enumerate( z.dtype.names ):
ks_D, ks_p = stats.ks_2samp( refData[ col ], z[ col ] )
min_p[ colNum ] = np.min(( min_p[ colNum ], ks_p ))
max_D[ colNum ] = np.max(( max_D[ colNum ], ks_D ))
dbg( 'i min_p max_D' )
def DefineRulesTo_fastCMS( pr, pops, chroms, selPop, sweepDir, cmsDir, genomeBuild = 'hg19' ):
"""Define rules to do fast CMS computation.
Params:
pr - the PipeRun object to which to add rules
selPop - testing selection in which pop?
pops - comparing selPop to which pops?
sweepDir - the sweep directory
cmsDir - the directory under which CMS stats go
"""
pops = list( MakeSeq( pops ) )
if selPop not in pops: pops.append( selPop )
allPops = tuple( MakeSeq( pops ) )
if selPop not in allPops: allPops += ( selPop, )
cmpPops = [ pop for pop in allPops if pop != selPop ]
rawScoresFN = {}
genMapSfx = genomeBuild2genMapSfx[ genomeBuild ]
for pop in allPops:
for chrom in chroms:
with pr.settingAttrs( 'pop chrom' ):
snpInfoFN = os.path.join( sweepDir, 'analysis/chr%(chrom)s/snps_%(pop)s.tsv' % locals() )
projDir = os.path.join( sweepDir, 'data/chr%(chrom)s' % locals() )
ancestralImportedFN = os.path.join( projDir, 'ancestral.tsv.imported' )
genotypesImportedFN = os.path.join( projDir, 'genotypes_chr%(chrom)s_%(pop)s_r21_nr_fwd_phased_all.imported' % locals() )
genMapImportedFN = os.path.join( projDir, 'genetic_map_chr%(chrom)s_%(genMapSfx)s.txt.imported' % locals() )
pr.addRule( name = 'extractSnpInfo',
commands = 'java -classpath ../Other/Ilya_Other/sweep/sweepsrc/sweep.jar edu.mit.broad.sweep.Main ExtractAlleleFreqs %(projDir)s/project %(snpInfoFN)s %(pop)s %(chrom)s' % locals(),
commandsOld = 'java -classpath ../Other/Ilya_Other/sweep/sweepsrc/sweep/target/sweep-1.0-SNAPSHOT-jar-with-dependencies.jar edu.mit.broad.sweep.Main ExtractAlleleFreqs %(projDir)s/project %(snpInfoFN)s %(pop)s %(chrom)s' % locals(),
depends_on = ( ancestralImportedFN, genotypesImportedFN, genMapImportedFN ),
creates = snpInfoFN )
chr2dihhFN = {}
for chrom in chroms:
with pr.settingAttrs( 'chrom' ):
chrom_s = 'chr' + str( chrom )
chromDir = os.path.join( cmsDir, chrom_s )
xpopScoresFN = os.path.join( chromDir, AddFileSfx( 'max_xpop.tsv', chrom_s, selPop, pops ) )
pr.addInvokeRule( invokeFn = gatherXPOPscores,
invokeArgs = Dict( 'pops chrom selPop sweepDir', outFN = xpopScoresFN ),
attrs = dict( pop = allPops, stat = 'max_xpop', piperun_short = True ) )
ihsFN = getFN_ihs_signif( **Dict( 'sweepDir chrom', pop = selPop ) )
ihsScoresFN = os.path.join( chromDir, AddFileSfx( 'iHS.tsv', chrom_s, selPop, pops ) )
dihhScoresFN = os.path.join( chromDir, AddFileSfx( 'dihh.tsv', chrom_s, selPop, pops ) )
chr2dihhFN[ chrom ] = dihhScoresFN
pop2ancFreqFN = os.path.join( cmsDir, chrom_s, AddFileSfx( 'pop2ancFreq.tsv', chrom_s, pops ) )
pop2sampleSizeFN = os.path.join( cmsDir, chrom_s, AddFileSfx( 'pop2sampleSize.tsv', chrom_s, pops ) )
pop2snpInfoFN = dict([ ( pop, os.path.join( sweepDir, 'analysis', chrom_s,
'snps_%(pop)s.tsv' % locals() ) )
for pop in pops ])
pr.addInvokeRule( invokeFn = gather_snp_info,
invokeArgs = Dict( 'pops pop2snpInfoFN pop2ancFreqFN pop2sampleSizeFN' ) )
pr.addInvokeRule( invokeFn = gather_iHS_scores,
invokeArgs = Dict( 'chrom selPop ihsFN pop2ancFreqFN',
# snpInfoFN = pop2snpInfoFN[ selPop ],
ihsOutFN = ihsScoresFN, dihhOutFN = dihhScoresFN ),
attrs = dict( pop = selPop, stat = ( 'iHS', 'StdDiff' ), piperun_short = True ) )
freqDiffScoresFN = os.path.join( chromDir, AddFileSfx( 'freqDiff.tsv', chrom_s, selPop, pops ) )
meanFstScoresFN = os.path.join( chromDir, AddFileSfx( 'meanFst.tsv', chrom_s, selPop, pops ) )
pr.addInvokeRule( invokeFn = computeMeanFstAndFreqDiffScores,
invokeArgs = Dict( 'chrom selPop sweepDir pops pop2ancFreqFN pop2sampleSizeFN',
outMeanFstFN = meanFstScoresFN,
outFreqDiffFN = freqDiffScoresFN ),
attrs = dict( pop = allPops, stat = ( 'freqDiff', 'meanFst' ), piperun_short = True ) )
StdDiffScoresFN = os.path.join( chromDir, AddFileSfx( 'StdDiff.tsv', chrom_s, selPop, pops ) )
rawScoresFN[ chrom ] = dict( iHS = ihsScoresFN, StdDiff = StdDiffScoresFN, meanFst = meanFstScoresFN,
freqDiff = freqDiffScoresFN,
max_xpop = xpopScoresFN )
# end: with pr.settingAttrs( 'chrom' )
# end: for chrom in chroms
# ihhStdFN = os.path.join( cmsDir, 'dihhstd.tsv' )
dihhGlobalStdFN = os.path.join( cmsDir, AddFileSfx( 'dihh_global_std.tsv', selPop, pops ) )
dihhBinMeansFN = os.path.join( cmsDir, AddFileSfx( 'dihh_bin_means.tsv', selPop, pops ) )
pr.addInvokeRule( invokeFn = normalizeByFreq_getMeanStd_tsv,
invokeArgs = dict( iHHDiffFNs = [ chr2dihhFN[k] for k in chroms ],
globalStatFN = dihhGlobalStdFN, binsStatFN = dihhBinMeansFN ),
name = 'compute_dihh_meanstd' )
# pr.addInvokeRule( invokeFn = computeMeanStd_binned_tsvs,
# invokeArgs = dict( inFNs = chr2dihhFN.values(), valCol = 'iHHDiff',
# binCol = 'normingFreqs', binMin = 0.05, binMax = 1.05, binStep = .05,
# outFN = ihhStdFN ),
# name = 'compute_dihh_std' )
for chrom in chroms:
with pr.settingAttrs( 'chrom' ):
chrom_s = 'chr' + str( chrom )
chromDir = os.path.join( cmsDir, chrom_s )
StdDiffScoresFN = os.path.join( chromDir, AddFileSfx( 'StdDiff.tsv', chrom_s, selPop, pops ) )
dbg( 'chrom chr2dihhFN[chrom]' )
pr.addInvokeRule( invokeFn = normalizeByFreq_compute_normed_tsv,
invokeArgs = dict( iHHDiffFN = chr2dihhFN[ chrom ],
globalStatFN = dihhGlobalStdFN,
binsStatFN = dihhBinMeansFN,
StdDiffFN = StdDiffScoresFN ) )
statFNs = {}
statLikesRatioFNs = {}
for stat in CMSBins.CMSstats:
with pr.settingAttrs( stat = stat, pop = ( selPop, ) if stat in ( 'iHS', 'StdDiff' ) else allPops, piperun_short = True ):
if stat not in CMSBins.nonNormedStats:
rawFNs = [ rawScoresFN[ chrom ][ stat ] for chrom in chroms ]
meanStdFN = os.path.join( cmsDir, AddFileSfx( 'meanStd.tsv', stat, selPop, pops ) )
# DefineRulesTo_computeMeanStd( pr, inFNs = rawFNs, colNum = 1,
# outFN = meanStdFN,
# addRuleArgs = \
# dict( name = 'computeMeanStd_for_stat',
# attrs = dict( chrom = chroms ) ) )
# meanStdBzFN = os.path.join( cmsDir, stat + '_meanStdForStat.tsv' )
pr.addInvokeRule( invokeFn = computeMeanStd,
invokeArgs = dict( inFNs = rawFNs, colName = stat, outFN = meanStdFN ) )
# end: if stat not in CMSBins.nonNormedStats
for chrom in chroms:
with pr.settingAttrs( 'chrom' ):
statFN = rawScoresFN[ chrom ][ stat ]
if stat not in CMSBins.nonNormedStats:
normedFN = AddFileSfx( statFN, 'normed' )
DefineRulesTo_normalizeOneColumn( pr, inFN = statFN,
meanStdFN = meanStdFN,
colName = stat,
outFN = normedFN,
addRuleArgs = dict( attrs = Dict( 'chrom' ) ) )
statFN = normedFN
bins_beg = CMSBins.stat_start[ stat ]
bins_end = CMSBins.stat_end[ stat ]
bins_n = CMSBins.stat_nbin[ stat ]
statFNs[ ( chrom, stat ) ] = statFN
statLikesRatioFN = AddFileSfx( rawScoresFN[ chrom ][ stat ], 'likesRatio' )
statLikesRatioFNs[ ( chrom, stat ) ] = statLikesRatioFN
pr.addInvokeRule( invokeFn = computeLikeRatioForStat,
invokeArgs = dict( stat = stat,
statValsFN = statFN,
hitLikesFN = '../Data/Common_Data/sim/likes/hitsLikes_toneutFixed_1.tsv',
missLikesFN = '../Data/Common_Data/sim/likes/missLikes_toneutFixed_1.tsv',
stat_start = bins_beg,
stat_end = bins_end,
stat_nbin = bins_n,
statLikesRatioFN = statLikesRatioFN ) )
# end: with pr.settingAttrs( 'chrom' )
# end: for chrom in chroms
# end: with pr.settingAttrs( stat = stat, piperun_short = True )
# end: for stat in CMSBins.CMSstats
for chrom in chroms:
with pr.settingAttrs( chrom = chrom, stat = CMSBins.CMSstats ):
chrom_s = 'chr' + str( chrom )
chromDir = os.path.join( cmsDir, chrom_s )
likesRatioFN = os.path.join( chromDir, AddFileSfx( 'likesRatio.tsv', CMSBins.CMSstats, selPop, pops ) )
pr.addInvokeRule( invokeFn = addLikesRatios,
invokeArgs = dict( inFNs = [ statLikesRatioFNs[ ( chrom, stat ) ] for stat in CMSBins.CMSstats ],
colNames = [ colName + 'likeRatio' for colName in CMSBins.CMSstats ],
outFN = likesRatioFN ) )
joinStatsFN = os.path.join( chromDir, AddFileSfx( 'joinStats.tsv', CMSBins.CMSstats, selPop, pops ) )
snpInfoFN = os.path.join( sweepDir, 'analysis/chr%(chrom)s/snps_%(selPop)s.tsv' % locals() )
pr.addInvokeRule( invokeFn = joinStats,
invokeArgs = dict( snpInfoFN = snpInfoFN,
statLikesFNs = [ statLikesRatioFNs[ ( chrom, stat ) ] for stat in CMSBins.CMSstats ],
likesRatioFN = likesRatioFN,
outFN = joinStatsFN ),
attrs = dict( stat = CMSBins.CMSstats, chrom = chrom ) )
def computeMeanStd_binned( inDatas, valCol, binCol, binMin, binMax, binStep ):
"""Compute binned stats for a set of tables"""
binCount = int( ( binMax - binMin ) / binStep )
dbg( 'binCount' )
sums = np.zeros( binCount )
sumsSq = np.zeros_like( sums )
counts = np.zeros_like( sums )
bins = np.arange( binMin, binMax, binStep )
for d_idx, d in enumerate( inDatas ):
dbg( 'd_idx d binStep' )
dbg( 'd[binCol]' )
binColValues = 1.0 - ( 1.0 - d[ binCol ].values )
for i in range( binCount ):
# binBot = bins[i]
binTop = bins[i]
theIdx = ( (binTop - binColValues) < binStep ) & ( ( binTop - binColValues ) > 0 )
# theIdx = ( binBot < d[ binCol ].values ) & ( d[ binCol ].values <= binTop )
# DotData( names = ('rows',), Columns = theIdx.nonzero() ).saveToSV( 'nz%02d.tsv' % i )
#rowsStr = ','.join(map(str,list(theIdx.nonzero())))
#print 'binnedRows=', rowsStr
hereVals = d[ theIdx ][ valCol ]
# DotData( names = ( 'temp', ), Columns = ( hereVals, ) ).saveToSV( 'temp2%2d.tsv' % i )
dbg( '"BEF" theIdx.sum() i bins[i] len(hereVals)' )
counts[i] += len( hereVals )
sums[i] += np.sum( hereVals )
sumsSq[i] += np.sum( hereVals * hereVals )
# dbg( '"AFT" i bins[i] bins[i+1] len(hereVals)' )
if False:
# fast version
binsHere = np.digitize( d[ binCol ], bins ) - 1
dbg( 'len(binsHere) binsHere' )
np.clip( binsHere, 0, binCount-1, out = binsHere );
dbg( 'binsHere' )
counts += np.bincount( binsHere, minlength = binCount )
sums += np.bincount( binsHere, weights = d[ valCol ], minlength = binCount )
sumsSq += np.bincount( binsHere, weights = d[ valCol ] * d[ valCol ], minlength = binCount )
countsOrig = counts.astype( int )
counts[ counts == 0 ] = np.nan
means = sums / counts
stds = sumsSq / counts - means * means
return pd.DataFrame( dict( binBeg = bins - binStep,
binEnd = bins,
counts = countsOrig, sums = sums, sumsSq = sumsSq,
means = means, stds = stds ) )
def localizeSpatiallyBySplineFitting( Ddata, scenario, nreplicas, thinSfx = '',
putativeMutPop = None, complikeSfx = '', likesTableSfx = '',
confidence = .9, minBins = 20, nbins = 200, smoothing = 0.0,
getio = None ):
"""For each replica within a given scenario,
localize the selected SNP spatially, by fitting a spline to a (smoothed version of) the CMS scores, dividing the
region into bins, and finding the set of bins that cover 90% (or specified fraction of) area under the spline.
Params:
confidence - the spatially localized region will (hopefully) have this probability of containing the causal SNP;
here, specifically, this means we'll include bins in the region that collectively cover this fraction of area
under the posterior density curve.
minBins - the region will include at least this many of the highest-average bins.
"""
snpStatsDir = os.path.join( Ddata, 'snpStats'+ thinSfx, scenario.scenDir() )
replicaStatsDir = os.path.join( Ddata, 'replicastats'+ thinSfx, scenario.scenDir() )
if putativeMutPop == None: putativeMutPop = scenario.mutPop
sfxs = ( putativeMutPop, complikeSfx, likesTableSfx )
complikeFN = os.path.join( snpStatsDir, AddFileSfx( 'complike.data/', *sfxs ) )
intervalsListFN = os.path.join( replicaStatsDir, AddFileSfx( 'intervalsSplineList.tsv', *sfxs ) )
intervalsStatsFN = os.path.join( replicaStatsDir, AddFileSfx( 'intervalsSplineStats.tsv', *sfxs ) )
posteriorSplineFN = os.path.join( replicaStatsDir, AddFileSfx( 'intervalsSplineSpline.tsv', *sfxs ) )
binInfoFN = os.path.join( replicaStatsDir, AddFileSfx( 'intervalsSplineBinInfo.tsv', *sfxs ) )
if getio:
return dict( depends_on = complikeFN,
creates = ( intervalsListFN, intervalsStatsFN, posteriorSplineFN, binInfoFN ),
mediumRuleNameSfx = ( scenario.scenDir(), ) + sfxs,
fileDescrs = { intervalsListFN:
'List of intervals in the region, one of which (hopefully) contains the causal SNP.'
' For each replica this table has one or more lines, giving intervals in that replica.',
intervalsStatsFN: 'Per-replica statistic about confidence intervals in that replica',
posteriorSplineFN: 'The results of interpolating posterior density as a spline',
binInfoFN: 'Information about individual bins under the spline' } )
complike = IDotData( complikeFN ).filter( lambda r: all( np.isfinite( ( r.iHS, r.meanFst, r.max_xpop ) ) ) )
binInfoHeadings = 'replicaNum binNum binStart binEnd included binCenters binAvgCMS binMaxCMS binIntegral '\
'binIntegralNormed binRank'
nbins_orig = nbins
with contextlib.nested( IDotData.openForWrite( posteriorSplineFN, 'replicaNum gdPos complikeExp' ),
IDotData.openForWrite( binInfoFN, binInfoHeadings ),
IDotData.openForWrite( intervalsListFN, 'replicaNum gdFrom gdTo gdSize bpFrom bpTo '
'bpSize binsInInterval binsArea binsMax binsMaxFrac binsAvg '
'binsMinRank binsMaxRank' ),
IDotData.openForWrite( intervalsStatsFN, 'replicaNum numSegments gdTotLen bpTotLen' ) ) \
as ( posteriorSplineFile, binInfoFile, intervalsListFile, intervalsStatsFile ):
for replicaNum, complikeForReplica in complike.groupby( 'Chrom' ):
gdMin = np.min( complikeForReplica.gdPos )
gdMax = np.max( complikeForReplica.gdPos )
binSize = ( gdMax - gdMin ) / nbins_orig
binLefts = np.arange( gdMin, gdMax, binSize )
binCenters = binLefts + binSize / 2
binRights = binLefts + binSize
dbg( 'len(complikeForReplica) replicaNum len(binLefts) len(binRights) len(binCenters)' )
nbins = len( binLefts )
#
# Compute the mean and max CMS score in each gdPos bin
#
binAvgCMS = np.zeros( nbins )
binMaxCMS = np.zeros( nbins )
binNums = np.zeros( nbins, dtype = int )
for bin, valsInBin in \
complikeForReplica.addCol( 'bin',
map( functools.partial( min, nbins-1 ),
map( int,
( complikeForReplica.gdPos - gdMin ) / binSize ))).groupby('bin'):
binNums[ bin ] = bin
if valsInBin:
binAvgCMS[ bin ] = np.mean( valsInBin.complikeExp )
binMaxCMS[ bin ] = max( valsInBin.complikeExp )
else:
binAvgCMS[ bin ] = binAvgCMS[ bin-1 ]
binMaxCMS[ bin ] = binMaxCMS[ bin-1 ]
# fit a spline to the function ( binCenter, binAvgCMS ), approximating the posterior probability of a SNP being
# causal.
posteriorDensitySpline = interpolate.splrep( binCenters, binAvgCMS, s = smoothing )
splineX = np.arange( gdMin, gdMax, binSize / 2 )
for x, y in zip( splineX, interpolate.splev( splineX, posteriorDensitySpline ) ):
posteriorSplineFile.writeRecord( replicaNum, x, y )
# compute the integral under the interpolated function, for each bin.
binIntegral = np.zeros( nbins )
for binLeft, binRight, binNum in zip( binLefts, binRights, binNums ):
binIntegral[ binNum ] = interpolate.splint( binLeft, binRight, posteriorDensitySpline )
# normalize the integral value above each bin so that the total posterior density of being causal
# integrates to 1.0
binIntegralNormed = binIntegral / sum( binIntegral )
binsByIntegralSize = binIntegral.argsort()[::-1]
binRank = np.zeros( nbins )
for i in range( nbins ):
binRank[ binsByIntegralSize[ i ] ] = i
binsToUse = np.zeros( nbins, dtype = bool )
binsIncluded = 0
fractionCovered = 0.0
for bin in binsByIntegralSize:
if fractionCovered >= confidence and binsIncluded >= minBins: break
binsToUse[ bin ] = True
binsIncluded += 1
fractionCovered += binIntegralNormed[ bin ]
# list the confidence intervals for this replica,
# merging adjacent intervals.
gd2bp = interpolate.interp1d( complikeForReplica.gdPos, complikeForReplica.Pos )
gdMax = max( complikeForReplica.gdPos )
binInfo = IDotData( names = binInfoHeadings,
Columns = ( itertools.repeat( replicaNum, nbins ),
range( nbins ),
binLefts, binRights, binsToUse,
binCenters, binAvgCMS, binMaxCMS, binIntegral, binIntegralNormed, binRank
) )
binInfoFile.writeRecords( binInfo )
binsOverallMaxCMS = max( binInfo.binMaxCMS )
gdTotLen = 0.0
bpTotLen = 0
numSegments = 0
for included, bins in binInfo.groupby( 'included' ):
if included:
gdFrom = min( bins.binStart )
gdTo = min( max( bins.binEnd ), gdMax )
bpFrom = int( gd2bp( gdFrom ) )
bpTo = int( gd2bp( gdTo ) )
gdTotLen += ( gdTo - gdFrom )
bpTotLen += ( bpTo - bpFrom )
numSegments += 1
intervalsListFile.writeRecord( replicaNum, gdFrom, gdTo, gdTo - gdFrom, bpFrom, bpTo, bpTo - bpFrom,
len( bins ), sum( bins.binIntegralNormed ),
max( bins.binMaxCMS ), max( bins.binMaxCMS ) / binsOverallMaxCMS,
np.mean( bins.binAvgCMS ),
min( bins.binRank ), max( bins.binRank ) )
assert numSegments > 0 and bpTotLen > 0 and gdTotLen > 0.0
intervalsStatsFile.writeRecord( replicaNum, numSegments, gdTotLen, bpTotLen )
dbg( 'replicaNum numSegments gdTotLen bpTotLen' )
def localizeSpatiallyByWindows(Ddata,
scenario,
nreplicas,
thinSfx='',
putativeMutPop=None,
complikeSfx='',
likesTableSfx='',
threshold=.5,
numSNP=1,
minGdInEachDir=.05,
fromReplica=None,
toReplica=None,
getio=None):
"""
Spatially localize the selected variant for all replicas within a given scenario.
The approach is to start with the highest-scoring SNP, and move left and right from it in fixed-size windows
for as long as the windows contain at least 'numSNP' snps with score at least 'threshold'.
Adapted from Operations.Shari_Operations.localize.hapmap_regions_0615.plotRegions() .
"""
snpStatsDir = os.path.join(Ddata, 'snpStats' + thinSfx, scenario.scenDir())
replicaStatsDir = os.path.join(Ddata, 'replicastats' + thinSfx,
scenario.scenDir())
if putativeMutPop == None: putativeMutPop = scenario.mutPop
sfxs = (putativeMutPop, complikeSfx, likesTableSfx)
complikeFN = os.path.join(snpStatsDir, AddFileSfx('complike.data/', *sfxs))
intervalsListFN = os.path.join(
replicaStatsDir, AddFileSfx('intervalsWindowsList.tsv', *sfxs))
if getio:
return dict(
depends_on=complikeFN,
creates=intervalsListFN,
mediumRuleNameSfx=(scenario.scenDir(), ) + sfxs,
fileDescrs={
intervalsListFN:
'List of intervals in the region, one of which (hopefully) contains the causal SNP.'
' For each replica this table has one or more lines, giving intervals in that replica.'
})
#complike = IDotData( complikeFN ).filter( lambda r: all( np.isfinite( ( r.iHS, r.meanFst, r.max_xpop ) ) ) )
complike = IDotData(complikeFN)
with IDotData.openForWrite(
intervalsListFN,
'replicaNum gdFrom gdTo gdSize bpFrom bpTo bpSize numPositiveBins '
'numSnpsOver_0_2 maxSNP_Pos maxSNP_lik') as intervalsListFile:
for replicaNum, complikeForReplica in complike.groupby('Chrom'):
if fromReplica is not None and replicaNum < fromReplica: continue
if toReplica is not None and replicaNum > toReplica: break
X = complikeForReplica.toDotData()
minPos = np.min(X.gdPos)
maxPos = np.max(X.gdPos)
bins = np.arange(0, 1, .01)
ind = X.complikeExp.argsort()
lik = X.complikeExp[ind]
maxlik = np.mean(lik[-5:])
#maxlik = mean(lik[-5:])
like = X.complikeExp / maxlik
Y = X[ind]
maxSNP = Y.Pos[-1]
maxScore = Y.complikeExp[-1]
relPos = X.gdPos - Y.gdPos[-1]
X = X.hstack(
DotData(Columns=[like, relPos],
names=['scaled_like', 'relPos']))
topGdPos = Y.gdPos[-1]
minPos = Y.Pos[-1]
maxPos = Y.Pos[-1]
minGdPos = topGdPos
maxGdPos = topGdPos
numPositiveBins = 0
dbg('replicaNum minPos maxPos minGdPos maxGdPos')
for dir in -1, +1:
for bin in bins:
Z = X[np.abs(X.relPos - dir * bin) <= .02]
dbg('dir bin len(Z)')
if len(Z) == 0: continue
top = Z[Z.scaled_like > threshold]
dbg('len(top)')
if len(top) <= numSNP and np.abs(
topGdPos - top.gdPos) > minGdInEachDir:
break
if len(top) == 0: top = Z
if dir == -1:
minPos = np.min(top.Pos)
minGdPos = np.min(top.gdPos)
else:
maxPos = np.max(top.Pos)
maxGdPos = np.max(top.gdPos)
numPositiveBins += 1
ind = np.all([X.Pos > minPos, X.Pos < maxPos], axis=0)
peak = X[ind]
intervalsListFile.writeRecord(replicaNum, minGdPos, maxGdPos,
maxGdPos - minGdPos, minPos, maxPos,
maxPos - minPos, numPositiveBins,
sum(peak.scaled_like > .2), maxSNP,
maxScore)
def likeVal(stat,hitsLikes,missLikes,stat_start,stat_end,nbin,lik,ind=None, defaultNumSnps = 10000,
useDefaultNumSnps = True):
"""For eah SNP, calculate loglikelihood of that SNP being causal based on one statistic,
given the SNP's bin for that statistic.
Input params:
stat - for each SNP, its statistic value (typically, normalized).
hitsLikes/missLikes - for each bin, what fraction of causal/non-causal SNPs has
the statistic in that bin?
stat_start, stat_end, nbin - bin boundaries and number of bins for the statistic
ind - optionally indicates which snps should be skipped. if given,
for snps for which ind[i] is false no likelihood value is computed
(and lik[i] is set to nan).
Output params:
lik - for each SNP, log of the probability that the SNP is causal given what bin this statistic
for this SNP falls into. the lik array is allocated by the caller, and filled-in
by this function.
Use by: complike()
"""
# if the (optional) indicator of which SNPs to compute likelihood for is not given,
# compute likelihood for all SNPs.
if ind == None or len(ind) < 1:
ind = ones(len(stat))
bin_size = float(stat_end - stat_start) / nbin
# var: p_causal - probability that a SNP chosen randomly from the replica is causal.
# within a replica (region), exactly one SNP is causal, so this is 1 / ( total number of SNPs ).
# For Bayes' rule, this is the prior probability that a SNP is causal, before we look at the value
# of the statistic.
#p_causal = 1. / len(stat)
numSnps = defaultNumSnps if useDefaultNumSnps else len( stat )
dbg( 'numSnps' )
p_causal = 1./numSnps if numSnps > 0 else np.nan
p_noncausal = 1 - p_causal
dbg( 'p_causal p_noncausal' )
for i in range(len(stat)):
if not ind[i]:
p_causal_bin = 0
elif not isfinite(stat[i]):
p_causal_bin = 1. / nbin
else:
bin = int((stat[i] - stat_start)/bin_size)
if bin >= nbin:
bin = nbin - 1
if bin < 0:
bin = 0
# P( causal | stat_in_this_bin ) = ( P( stat_in_this_bin | causal ) * P( causal ) ) / P( stat_in_this_bin )
# here, hitLikes[bin] gives P( stat_in_this_bin )
num = hitsLikes[bin]*p_causal
# var: denom - probability that a SNP, whether causal or not, has this statistic in this bin
denom = hitsLikes[bin] * p_causal + missLikes[bin]*p_noncausal
assert denom != 0
#print num,denom
p_causal_bin = num / denom
if np.abs( stat[i] - 4.6387211 ) < 1e-5:
dbg( '"here!" bin stat[i] hitsLikes[bin] p_causal num denom p_causal_bin' )
# endif: whether we have the statistic value for this snp,
# and whether we are asked to compute the SNP's likelihood of being causal
likscore = log(p_causal_bin) if p_causal_bin > 0.0 else nan
lik[i] = likscore
def RunTasks( options ):
"""Take tasks from the specified queue directory, and run them.
Params:
options - see command-line parameter definition in main() below.
"""
if haveParamiko: Random.atfork()
startClock = time.time()
logging.info( 'Starting runner (process id %d on host %s) with options %s'
% ( os.getpid(), GetHostName(), options ) )
stopSignal = [ False ]
def SetStopSignal( sigNum, stkFrm ):
logging.info( 'Setting stop signal to stop runners' )
stopSignal[ 0 ] = True
dbg( '"aftset" stopSignal' )
signal.signal( signal.SIGUSR1, SetStopSignal )
fs = RemoteFileSystem( remote = options.remote, pw = options.password, pkey = options.pkey ) \
if options.remote else LocalFileSystem()
# check that all queues exist
assert all( fs.exists( queue ) and fs.isdir( queue )
for queue in options.queues.split( fs.pathsep ) )
# register a cleanup routine so that, if we claim a task and then
# crash midway through it, our lock on the task is erased, so that
# another runner can pick up the task. Note that if the _task_ fails
# with an error code, that's fine -- we just report the error code
# to the mqsub.sh script instance that submitted the task.
# The cleanup here happens only if the runner crashes before receiving
# a proper exit code from the task.
fileToErase = [ None ]
@atexit.register
def DoErase( eraseWhat = fileToErase ):
if eraseWhat[0] and fs.exists(eraseWhat[0]):
#print 'runner AtExit: removing ' + eraseWhat[0]
fs.remove( eraseWhat[0] )
# var: lastFinish - time when a task last finished.
lastFinish = time.time()
queues = options.queues.split( fs.pathsep )
lastQueueModTime = [ None ] * len( queues )
skipDirs = set([ 'newtask.dat' ] )
numTasksRun = 0
numProcsAvail = int( os.getenv( 'LSB_DJOB_NUMPROC', 1 ) )
dbg( 'numProcsAvail' )
for queue in queues:
EnsureDirExists( os.path.join( queue, 'succ' ) )
EnsureDirExists( os.path.join( queue, 'fail' ) )
while not stopSignal[0]:
ranCommand = False
if options.maxRunHours > 0 and ( time.time() - startClock ) / 3600.0 > options.maxRunHours:
logging.info( 'Runner exiting after CPU time of %s hours' % ( time.time() - startClock ) / 3600.0 )
return
if stopSignal[ 0 ]:
logging.info( 'Runner stopped by stop signal' )
return
else: dbg( '"chkstop" stopSignal' )
dbg( 'queues' )
for queueNum, queue in enumerate( queues ):
dbg( 'queueNum queue' )
# do a quick check to see if any tasks have been added to the queue since we last checked
newTaskFN = os.path.join( queue, 'newtask.dat' )
try:
curQueueModTime = fs.stat( newTaskFN ).st_mtime
if curQueueModTime == lastQueueModTime[ queueNum ]: continue
lastQueueModTime[ queueNum ] = curQueueModTime
except EnvironmentError as e:
if os.path.exists( newTaskFN ):
logging.warning( 'ERROR CHECKING FOR NEW TASKS in queue %s: %s' % ( queue, e ) )
pass
# find an unclaimed task in this queue, and try to claim it
taskDirs = sorted( fs.listdir( queue ) )
dbg( 'len(taskDirs)' )
#random.shuffle( taskDirs )
dbg( 'os.environ.get("MQ_FIRST_DIR")' )
if 'MQ_FIRST_DIR' in os.environ and os.environ[ 'MQ_FIRST_DIR' ] in taskDirs:
taskDirs = [ os.environ[ 'MQ_FIRST_DIR' ] ] + taskDirs
logging.info( 'putting specified dir first' )
for taskDir in taskDirs:
if taskDir in skipDirs: continue
if options.maxRunHours > 0 and ( ( time.time() - startClock ) / 3600.0 ) > options.maxRunHours:
logging.info( 'Runner exiting after CPU time of %s hours' % ( ( time.time() - startClock ) / 3600.0 ) )
return
if stopSignal[ 0 ]:
logging.info( 'Runner stopped by stop signal' )
return
else: dbg( '"chkstop" stopSignal' )
try:
while fs.path.exists( os.path.join( queue, 'noclaim.dat' ) ):
time.sleep( 60 + random.normalvariate( 10.0, 5.0 ) )
fullTaskDir = fs.path.join( queue, taskDir )
claimedFN = fs.path.join( fullTaskDir, options.claimedFN )
attrsFN = fs.path.join( fullTaskDir, 'attrs.tsv' )
cwdFN = fs.path.join( fullTaskDir, 'submitdir.txt' )
failedCond = []
def saveVal( name, val, fc = failedCond ):
if not val: fc.append( name )
return val
if saveVal( 'ready', fs.path.exists( fs.path.join( fullTaskDir, options.readyFN ) ) ) \
and saveVal( 'not claimed', not fs.path.exists( claimedFN ) ) \
and saveVal( 'relocatable', ( not options.remote or \
all([ not f.startswith( '/' ) for which in ( 'sources', 'targets' ) \
for f in fs.SlurpFile( fs.path.join( fullTaskDir, which + '.lst' ) ) ]) ) ) \
and saveVal( 'memOk', GetMemReq( fs, attrsFN ) <= options.maxMem ) \
and saveVal( 'minMemOk', options.minMem == 0 or GetMemReq( fs, attrsFN ) >= options.minMem ) \
and saveVal( 'minProc', GetProcReq( fs, attrsFN ) >= options.minProc ) \
and saveVal( 'maxProc', GetProcReq( fs, attrsFN ) <= numProcsAvail ) \
and saveVal( 'local', ( options.local_tasks or not GetTaskAttr( fs, attrsFN, 'piperun_run_locally', False ) ) ) \
and saveVal( 'onlyLocal',
( not options.only_local_tasks or GetTaskAttr( fs, attrsFN, 'piperun_run_locally', False ) ) ) \
and saveVal( 'short', ( not options.runOnlyShort or GetTaskAttr( fs, attrsFN, 'piperun_short', False ) ) ) \
and saveVal( 'long', ( not options.runOnlyLong or not GetTaskAttr( fs, attrsFN, 'piperun_short', False ) ) ) \
and saveVal( 'notRequeued', ( not options.noRequeuedTasks or not fs.path.exists( fs.path.join( fullTaskDir, 'requeued.dat' ) ) ) ) \
and saveVal( 'notFromHost', ( not options.onlyFromHost or socket.getfqdn() == options.onlyFromHost ) ) \
and saveVal( 'notFromPipeline', ( not options.onlyFromPipelineId or \
GetTaskAttr( fs, attrsFN, 'piperun_pipelineId' ) == options.onlyFromPipelineId ) ):
# try to claim the task
try:
fd = fs.open(claimedFN, os.O_CREAT|os.O_EXCL|os.O_WRONLY)
except EnvironmentError:
# another runner beat us to this task -- go and check other tasks
logging.info( 'another job beat us to claiming ' + fullTaskDir )
continue
try:
fs.write( fd, 'locked by process %d on host %s\n' % ( os.getpid(), GetHostName() ) )
for v in os.environ.keys():
fs.write( fd, '%s=%s\n' % ( v, os.environ[v] ) )
finally:
fs.close( fd )
# Tell our cleanup code to release this task if we crash.
fileToErase[0] = claimedFN
# get the command to run the task
theCMD = fs.SlurpFile( os.path.join( fullTaskDir, 'command.dat' ) ).strip()
theCmdDir = fs.SlurpFile( os.path.join( fullTaskDir, 'submitdir.txt' ) ).strip()
theCmdEnvFN = os.path.join( fullTaskDir, 'submitenv.txt' )
if options.remote:
assert have_fcntl
SystemSucceed( 'mkdir -p ' + os.path.join( options.localDataDir, fs.root[1:] ) )
for needDir in 'Operations', 'Classes', 'System', 'Other':
needDirFull = os.path.join( options.localDataDir, fs.root[1:], '..', needDir )
if not os.path.exists( needDirFull ):
os.symlink( os.path.realpath( os.path.join( '..', needDir ) ), needDirFull )
# copy source files
# get exclusive locks on the source files
srcFiles = sorted( set( fs.SlurpFile( os.path.join( fs.root, fullTaskDir, 'sources.lst' ) ).rstrip( '\n' ).split( '\n' ) ) )
srcLockIds = []
srcLockFiles = []
for srcFile in srcFiles:
lockFile = os.path.join( options.localDataDir, 'mqlocks', srcFile[1:] )
if lockFile.endswith('/'): lockFile = lockFile[:-1]
lockFile += '.lock'
SystemSucceed( 'mkdir -p ' + os.path.dirname( lockFile ) )
gotLock = False
while not gotLock:
try:
openMode = os.O_CREAT|os.O_EXCL|os.O_WRONLY
logging.info( 'opening ' + lockFile + ' with mode ' + str( openMode ) )
lockId = os.open( lockFile, openMode )
gotLock = True
except EnvironmentError:
logging.info( 'Could not create ' + lockFile + ' , waiting...' )
time.sleep( 10 + random.normalvariate( 3.0, 1.0 ) )
fcntl.lockf( lockId, fcntl.LOCK_EX )
srcLockIds.append( lockId )
srcLockFiles.append( lockFile )
logging.info( 'Got lock on ' + lockFile )
SystemSucceed( 'rsync -zprv --files-from=:' + os.path.join( fs.root, fullTaskDir, 'sources.lst' ) +
' ' + fs.username + '@' + fs.hostname + ':/ ' + options.localDataDir )
for srcLockId, srcLockFile in zip( srcLockIds, srcLockFiles)[::-1]:
fcntl.lockf( srcLockId, fcntl.LOCK_UN )
os.close( srcLockId )
SystemSucceed( 'rm -rf ' + srcLockFile )
targets = fs.SlurpFile( os.path.join( fs.root, fullTaskDir, 'targets.lst' ) ).rstrip( '\n' ).split('\n')
targetDirs = set( map( os.path.dirname, filter( None, map( str.strip, targets ) ) ) )
dbg( '"DDDDDD" targetDirs' )
for targetDir in targetDirs:
assert targetDir.startswith( '/' )
tdir = os.path.join( options.localDataDir, targetDir[1:] )
SystemSucceed( 'mkdir -p ' + tdir + ' ' + os.path.join( tdir, 'makeinfo' ) )
theCMD = 'cd ' + os.path.join( options.localDataDir, fs.root[1:] ) + ' && ' + theCMD
logging.info( 'Under ' + claimedFN + ' RUNNING: ' + theCMD )
# Actually run the task; get its exit code
save_cwd = os.getcwd()
try:
os.chdir( theCmdDir )
logging.info( 'CWD=' + os.getcwd() )
runScriptFN = os.path.join( fullTaskDir, 'run.sh' )
with open( runScriptFN, 'w' ) as out:
out.write( '#!/usr/bin/env bash\n' )
out.write( 'set -e -o pipefail\n' )
with open( theCmdEnvFN ) as envFile:
for line in envFile:
if '=' not in line or line.startswith('module='): break
equalIdx = line.index( '=' )
envVarName = line[ :equalIdx+1 ]
if not ( re.search( r'\W', envVarName ) or envVarName.startswith( 'LSB_' ) or \
envVarName.startswith( 'LSF_' ) or \
envVarName.startswith( 'LS_' ) or envVarName.startswith( 'SLURM' ) or \
envVarName in \
( 'SYS_TYPE', 'MACHTYPE', 'VENDOR', 'OSTYPE',
'DOMAINNAME', 'HOSTTYPE', 'SHORTHOST', 'SSH_TTY',
'HOST', 'HOSTNAME', 'REMOTEHOST', 'STY' ) ):
out.write( 'export ' + envVarName + "'" + line[ equalIdx+1: -1 ] + "'\n" )
out.write( theCMD )
os.chmod( runScriptFN, stat.S_IXUSR | stat.S_IRWXU )
try:
exitCode = os.system( runScriptFN )
except ( KeyboardInterrupt, SystemExit ):
interruptedFN = os.path.join( fullTaskDir, 'interrupted.dat' )
DumpFile( interruptedFN, 'interrupted' );
raise
finally:
os.chdir( save_cwd )
logging.info( 'Under ' + claimedFN + ' FINISHED RUNNING: ' + theCMD )
logging.info( 'Got exit code %d' % exitCode )
if options.remote:
# copy the target files and the output log back to the correct dirs on the remote system
# first, make sure the files all exist, and are no longer being written to.
time.sleep( options.aftTaskDelay )
os.system( 'rsync -zprv --files-from=:' + os.path.join( fs.root, fullTaskDir, 'targets.lst' ) +
' ' + options.localDataDir + ' ' + fs.username + '@' + fs.hostname + ':/' )
# If we succeeded in running the task (whether the task itself failed or not),
# tell the cleanup code to NOT release this task if we crash.
fileToErase[0] = None
# Tell the task submitter script that we are done, and what the task's
# exit code was.
if os.path.exists( os.path.join( fullTaskDir, 'nmq.dat' ) ):
time.sleep(3)
fd = fs.open( os.path.join( fullTaskDir, 'completed.dat' ), os.O_CREAT|os.O_EXCL|os.O_WRONLY )
fs.close(fd)
try:
shutil.move( fullTaskDir, os.path.join( queue, 'succ' if exitCode == 0 else 'fail' ) )
except EnvironmentError as e:
logging.warning( 'Error moving ' + fullTaskDir + ' to ' + os.path.join( queue, 'succ' if exitCode == 0 else 'fail' ) + ' : ' + e )
else:
exitCodeFN = os.path.join( fullTaskDir, 'exitCode.dat' )
fd = fs.open( exitCodeFN, os.O_CREAT|os.O_EXCL|os.O_WRONLY )
bytesWritten = fs.write( fd, str( exitCode ) )
fs.close( fd )
time.sleep(3)
logging.info( 'Wrote exit code %s to file %s (%s bytes)' % ( exitCode, exitCodeFN, bytesWritten ) )
fd = fs.open( os.path.join( fullTaskDir, 'completed.dat' ), os.O_CREAT|os.O_EXCL|os.O_WRONLY )
fs.close(fd)
# Record that we actually ran a task here.
ranCommand = True
lastFinish = time.time()
numTasksRun += 1
else:
logging.info( 'did not take task ' + taskDir + ' ; reason: ' + str( failedCond ) );
except:
excInfo = sys.exc_info()
logging.warning( 'Error trying to grab task from ' + taskDir + ' (%s), skipping...'
% str( excInfo ) )
traceback.print_exc()
dbg( 'ranCommand lastFinish time.time()-lastFinish' )
if not ranCommand:
waitTimeHere = time.time() - lastFinish
if ( numTasksRun > 0 and options.maxWaitTime > 0 and waitTimeHere > options.maxWaitTime ) \
or ( numTasksRun == 0 and options.maxFirstWaitTime > 0 and waitTimeHere > options.maxFirstWaitTime ) :
logging.info( 'Runner exiting after idle time of %s' % waitTimeHere )
return
time.sleep( options.taskCheckInterval + random.normalvariate( 3.0, 1.0 ) )
def DefineRulesTo_RunSweepOnSims(pr,
Ddata,
simsOut,
thinExt='',
thinning='',
suffix='',
mutAges=AllAges,
mutPops=AllPops,
mutFreqs=AllFreqs,
nreplicas=100,
pop2name=pop2name,
tests=('lrh', 'ihs', 'xpop'),
acceptExistingSimConfigFiles=False,
setOptions=(),
appendOptions=(),
inputParamsFiles=[],
runImportsLocally=True,
noImports=False,
powerSfx='',
doOnlyStages=None):
"""Define rules for running simulations and doing Sweep analyses of them.
Parameters:
mutAges, mutPops, mutFreqs - parameters defining the selection scenario.
"""
# Define the rules to do Sweep analyses of the simulations.
# These rules are created by running a Perl script, sim_analysis_pipe.pl .
# Each invocation of the script defines rules for one type of test.
# The rules for each test are saved to an .xml file, and then all these files are
# merged into pr.
# Rather than explicitly invoking the script three times we define a small pipeline to do this.
# The output of this pipeline is a pipeline definition for analyzing simulation results by each of the tests,
# saved to an .xml file.
simPipeline = PipeRun(name='DefineSims',
descr='Define simulation and analysis pipeline')
if not powerSfx: powerSfx = Sfx(*mutAges)
if not acceptExistingSimConfigFiles:
simPipeline.addInvokeRule(
invokeFn=WriteSimulationInfo,
invokeArgs=Dict('Ddata mutAges mutPops mutFreqs nreplicas suffix '
'inputParamsFiles pop2name powerSfx'))
test2pipeline = {}
for test in tests:
simSfx = ''
if suffix: simSfx += suffix
if thinning: simSfx += thinning
thinExtHere = '' if thinning else thinExt
testPipeline = os.path.join('Ilya_Temp',
AddFileSfx('p.xml', pr.name, test))
test2pipeline[test] = testPipeline
if thinning:
simPipeline.addRule(
comment="Copy config file",
targets="$Ddata/power_$test$simSfx/config$powerSfx.txt",
sources="$Ddata/power_$test$suffix/config.txt",
commands="cp $Ddata/power_$test$suffix/config.txt "
"$Ddata/power_$test$simSfx/config$powerSfx.txt",
mediumRuleName='copy_config_' + test,
name='copy_config')
simPipeline.addRule(
targets=testPipeline,
sources=[
'../Other/Ilya_Other/sweep/sims/scripts/sim_analysis_pipe.pl',
"$Ddata/power_$test$simSfx/config$powerSfx.txt",
"$Ddata/config$suffix/sims$powerSfx.txt",
"$Ddata/config$suffix/scenarios$powerSfx.txt",
"$Ddata/config$suffix/pops$powerSfx.txt"
],
commands=
"../Other/Ilya_Other/sweep/sims/scripts/sim_analysis_pipe.pl "
" --only-write-pipeline $testPipeline " +
('--run-imports-locally ' if runImportsLocally else '') +
('--no-imports ' if noImports else '') +
(('--do-only-stages ' + doOnlyStages +
' ') if doOnlyStages else '') +
('--sim-suffix ' + simSfx if simSfx else '') +
('--powerSfx ' + powerSfx if powerSfx else '') +
(' --thin-ext ' + thinExtHere if thinExtHere else '') +
reduce(operator.concat, [
' --set-option ' + setOption[0] + ' ' + setOption[1]
for setOption in setOptions
], '') + reduce(operator.concat, [
' --append-option ' + appendOption[0] + " '" +
appendOption[1] + "'" for appendOption in appendOptions
], '') +
" --target-test $test $Ddata/$simsOut$simSfx $Ddata/power_$test$simSfx",
name='simanal',
mediumRuleName='simanal_$test$simSfx',
comment='Define rules for analyzing simulations')
dbg('"RUNNING_simPipeline"')
simPipeline.runForced(aftRuleDelay=5)
dbg('"DONE_RUNNING_simPipeline"')
for test in tests:
pr.addPipelineFromFile(test2pipeline[test])
def fst_onePopPair(ancFreqs, sampleSizes):
"""Compute fst between two pops, for each SNP, given the ancestral freq in each pop and the sample sizes.
"""
dbg( 'ancFreqs sampleSizes' )
n = sampleSizes
n_tot = n[0] + n[1]
# nanc = np.ceil([ ancFreqs[0] * sampleSizes[0], ancFreqs[1] * sampleSizes[1] ])
nanc = ( np.array([ ancFreqs[0] * sampleSizes[0], ancFreqs[1] * sampleSizes[1] ]) + .5 ).astype( int )
dbg( 'nanc' )
f1 = ancFreqs
f2 = 1 - ancFreqs
dbg( 'f1 f2 f1.shape f2.shape' )
# Use Weir-Hill estimator for Fst
pmean = (nanc[0] + nanc[1]) / n_tot
nic = n[0] - (n[0]*n[0])/n_tot
njc = n[1] - (n[1]*n[1])/n_tot
dbg( 'pmean.shape nic.shape njc.shape nic njc' )
#assert nic == njc == 60
nc = nic + njc
#print nc, f1[0], f1[1]
#print f2[0], f2[1]
msp = n[0] * (f1[0] - pmean) * (f1[0] - pmean) \
+ n[1] * (f1[1] - pmean) * (f1[1] - pmean)
msg = ((n[0] * f1[0]* f2[0]) + (n[1] * f1[1] * f2[1])) \
/ (n[0] - 1 + n[1] - 1)
dbg( 'msp.shape msg.shape' )
num = msp - msg
denom = msp + (msg*(nc - 1))
#print msp, msg
#print num, denom
dbg( 'num.shape denom.shape' )
denom [ denom == 0 ] = nan
an_fst = num / denom
dbg( 'an_fst.shape' )
ipop = 0
jpop = 1
dbg( '"KKKKKKKKKKKKK" pmean nanc[ipop] nanc[jpop] n[ipop] n[jpop] nic njc nc msp msg num denom an_fst' )
return an_fst
def localizeSpatiallyByWindows(Ddata, scenario, nreplicas, thinSfx = '', putativeMutPop = None, complikeSfx = '',
likesTableSfx = '',
threshold = .5, numSNP = 1,
minGdInEachDir = .05,
fromReplica = None, toReplica = None, getio = None):
"""
Spatially localize the selected variant for all replicas within a given scenario.
The approach is to start with the highest-scoring SNP, and move left and right from it in fixed-size windows
for as long as the windows contain at least 'numSNP' snps with score at least 'threshold'.
Adapted from Operations.Shari_Operations.localize.hapmap_regions_0615.plotRegions() .
"""
snpStatsDir = os.path.join( Ddata, 'snpStats'+ thinSfx, scenario.scenDir() )
replicaStatsDir = os.path.join( Ddata, 'replicastats'+ thinSfx, scenario.scenDir() )
if putativeMutPop == None: putativeMutPop = scenario.mutPop
sfxs = ( putativeMutPop, complikeSfx, likesTableSfx )
complikeFN = os.path.join( snpStatsDir, AddFileSfx( 'complike.data/', *sfxs ) )
intervalsListFN = os.path.join( replicaStatsDir, AddFileSfx( 'intervalsWindowsList.tsv', *sfxs ) )
if getio:
return dict( depends_on = complikeFN,
creates = intervalsListFN,
mediumRuleNameSfx = ( scenario.scenDir(), ) + sfxs,
fileDescrs =
{ intervalsListFN:
'List of intervals in the region, one of which (hopefully) contains the causal SNP.'
' For each replica this table has one or more lines, giving intervals in that replica.' } )
#complike = IDotData( complikeFN ).filter( lambda r: all( np.isfinite( ( r.iHS, r.meanFst, r.max_xpop ) ) ) )
complike = IDotData( complikeFN )
with IDotData.openForWrite( intervalsListFN,
'replicaNum gdFrom gdTo gdSize bpFrom bpTo bpSize numPositiveBins '
'numSnpsOver_0_2 maxSNP_Pos maxSNP_lik' ) as intervalsListFile:
for replicaNum, complikeForReplica in complike.groupby( 'Chrom' ):
if fromReplica is not None and replicaNum < fromReplica: continue
if toReplica is not None and replicaNum > toReplica: break
X = complikeForReplica.toDotData()
minPos = np.min(X.gdPos)
maxPos = np.max(X.gdPos)
bins = np.arange(0,1,.01)
ind = X.complikeExp.argsort()
lik = X.complikeExp[ind]
maxlik = np.mean(lik[-5:])
#maxlik = mean(lik[-5:])
like = X.complikeExp / maxlik
Y = X[ind]
maxSNP = Y.Pos[-1]
maxScore = Y.complikeExp[-1]
relPos = X.gdPos - Y.gdPos[-1]
X = X.hstack(DotData(Columns = [like,relPos],names=['scaled_like','relPos']))
topGdPos = Y.gdPos[-1]
minPos = Y.Pos[-1]
maxPos = Y.Pos[-1]
minGdPos = topGdPos
maxGdPos = topGdPos
numPositiveBins = 0
dbg( 'replicaNum minPos maxPos minGdPos maxGdPos' )
for dir in -1, +1:
for bin in bins:
Z = X[np.abs(X.relPos - dir*bin) <= .02]
dbg( 'dir bin len(Z)' )
if len(Z) == 0: continue
top = Z[Z.scaled_like > threshold]
dbg( 'len(top)' )
if len(top) <= numSNP and np.abs( topGdPos - top.gdPos ) > minGdInEachDir: break
if len( top ) == 0: top = Z
if dir == -1:
minPos = np.min(top.Pos)
minGdPos = np.min(top.gdPos)
else:
maxPos = np.max(top.Pos)
maxGdPos = np.max(top.gdPos)
numPositiveBins += 1
ind = np.all([X.Pos > minPos, X.Pos < maxPos],axis=0)
peak = X[ind]
intervalsListFile.writeRecord( replicaNum, minGdPos, maxGdPos, maxGdPos - minGdPos,
minPos, maxPos, maxPos - minPos,
numPositiveBins, sum(peak.scaled_like > .2),
maxSNP, maxScore )
def localizeSpatiallyBySplineFitting(Ddata,
scenario,
nreplicas,
thinSfx='',
putativeMutPop=None,
complikeSfx='',
likesTableSfx='',
confidence=.9,
minBins=20,
nbins=200,
smoothing=0.0,
getio=None):
"""For each replica within a given scenario,
localize the selected SNP spatially, by fitting a spline to a (smoothed version of) the CMS scores, dividing the
region into bins, and finding the set of bins that cover 90% (or specified fraction of) area under the spline.
Params:
confidence - the spatially localized region will (hopefully) have this probability of containing the causal SNP;
here, specifically, this means we'll include bins in the region that collectively cover this fraction of area
under the posterior density curve.
minBins - the region will include at least this many of the highest-average bins.
"""
snpStatsDir = os.path.join(Ddata, 'snpStats' + thinSfx, scenario.scenDir())
replicaStatsDir = os.path.join(Ddata, 'replicastats' + thinSfx,
scenario.scenDir())
if putativeMutPop == None: putativeMutPop = scenario.mutPop
sfxs = (putativeMutPop, complikeSfx, likesTableSfx)
complikeFN = os.path.join(snpStatsDir, AddFileSfx('complike.data/', *sfxs))
intervalsListFN = os.path.join(
replicaStatsDir, AddFileSfx('intervalsSplineList.tsv', *sfxs))
intervalsStatsFN = os.path.join(
replicaStatsDir, AddFileSfx('intervalsSplineStats.tsv', *sfxs))
posteriorSplineFN = os.path.join(
replicaStatsDir, AddFileSfx('intervalsSplineSpline.tsv', *sfxs))
binInfoFN = os.path.join(replicaStatsDir,
AddFileSfx('intervalsSplineBinInfo.tsv', *sfxs))
if getio:
return dict(
depends_on=complikeFN,
creates=(intervalsListFN, intervalsStatsFN, posteriorSplineFN,
binInfoFN),
mediumRuleNameSfx=(scenario.scenDir(), ) + sfxs,
fileDescrs={
intervalsListFN:
'List of intervals in the region, one of which (hopefully) contains the causal SNP.'
' For each replica this table has one or more lines, giving intervals in that replica.',
intervalsStatsFN:
'Per-replica statistic about confidence intervals in that replica',
posteriorSplineFN:
'The results of interpolating posterior density as a spline',
binInfoFN:
'Information about individual bins under the spline'
})
complike = IDotData(complikeFN).filter(
lambda r: all(np.isfinite((r.iHS, r.meanFst, r.max_xpop))))
binInfoHeadings = 'replicaNum binNum binStart binEnd included binCenters binAvgCMS binMaxCMS binIntegral '\
'binIntegralNormed binRank'
nbins_orig = nbins
with contextlib.nested( IDotData.openForWrite( posteriorSplineFN, 'replicaNum gdPos complikeExp' ),
IDotData.openForWrite( binInfoFN, binInfoHeadings ),
IDotData.openForWrite( intervalsListFN, 'replicaNum gdFrom gdTo gdSize bpFrom bpTo '
'bpSize binsInInterval binsArea binsMax binsMaxFrac binsAvg '
'binsMinRank binsMaxRank' ),
IDotData.openForWrite( intervalsStatsFN, 'replicaNum numSegments gdTotLen bpTotLen' ) ) \
as ( posteriorSplineFile, binInfoFile, intervalsListFile, intervalsStatsFile ):
for replicaNum, complikeForReplica in complike.groupby('Chrom'):
gdMin = np.min(complikeForReplica.gdPos)
gdMax = np.max(complikeForReplica.gdPos)
binSize = (gdMax - gdMin) / nbins_orig
binLefts = np.arange(gdMin, gdMax, binSize)
binCenters = binLefts + binSize / 2
binRights = binLefts + binSize
dbg('len(complikeForReplica) replicaNum len(binLefts) len(binRights) len(binCenters)'
)
nbins = len(binLefts)
#
# Compute the mean and max CMS score in each gdPos bin
#
binAvgCMS = np.zeros(nbins)
binMaxCMS = np.zeros(nbins)
binNums = np.zeros(nbins, dtype=int)
for bin, valsInBin in \
complikeForReplica.addCol( 'bin',
list(map( functools.partial( min, nbins-1 ),
list(map( int,
( complikeForReplica.gdPos - gdMin ) / binSize ))))).groupby('bin'):
binNums[bin] = bin
if valsInBin:
binAvgCMS[bin] = np.mean(valsInBin.complikeExp)
binMaxCMS[bin] = max(valsInBin.complikeExp)
else:
binAvgCMS[bin] = binAvgCMS[bin - 1]
binMaxCMS[bin] = binMaxCMS[bin - 1]
# fit a spline to the function ( binCenter, binAvgCMS ), approximating the posterior probability of a SNP being
# causal.
posteriorDensitySpline = interpolate.splrep(binCenters,
binAvgCMS,
s=smoothing)
splineX = np.arange(gdMin, gdMax, binSize / 2)
for x, y in zip(splineX,
interpolate.splev(splineX,
posteriorDensitySpline)):
posteriorSplineFile.writeRecord(replicaNum, x, y)
# compute the integral under the interpolated function, for each bin.
binIntegral = np.zeros(nbins)
for binLeft, binRight, binNum in zip(binLefts, binRights, binNums):
binIntegral[binNum] = interpolate.splint(
binLeft, binRight, posteriorDensitySpline)
# normalize the integral value above each bin so that the total posterior density of being causal
# integrates to 1.0
binIntegralNormed = binIntegral / sum(binIntegral)
binsByIntegralSize = binIntegral.argsort()[::-1]
binRank = np.zeros(nbins)
for i in range(nbins):
binRank[binsByIntegralSize[i]] = i
binsToUse = np.zeros(nbins, dtype=bool)
binsIncluded = 0
fractionCovered = 0.0
for bin in binsByIntegralSize:
if fractionCovered >= confidence and binsIncluded >= minBins:
break
binsToUse[bin] = True
binsIncluded += 1
fractionCovered += binIntegralNormed[bin]
# list the confidence intervals for this replica,
# merging adjacent intervals.
gd2bp = interpolate.interp1d(complikeForReplica.gdPos,
complikeForReplica.Pos)
gdMax = max(complikeForReplica.gdPos)
binInfo = IDotData(names=binInfoHeadings,
Columns=(itertools.repeat(replicaNum, nbins),
list(range(nbins)), binLefts,
binRights, binsToUse, binCenters,
binAvgCMS, binMaxCMS, binIntegral,
binIntegralNormed, binRank))
binInfoFile.writeRecords(binInfo)
binsOverallMaxCMS = max(binInfo.binMaxCMS)
gdTotLen = 0.0
bpTotLen = 0
numSegments = 0
for included, bins in binInfo.groupby('included'):
if included:
gdFrom = min(bins.binStart)
gdTo = min(max(bins.binEnd), gdMax)
bpFrom = int(gd2bp(gdFrom))
bpTo = int(gd2bp(gdTo))
gdTotLen += (gdTo - gdFrom)
bpTotLen += (bpTo - bpFrom)
numSegments += 1
intervalsListFile.writeRecord(
replicaNum, gdFrom, gdTo, gdTo - gdFrom, bpFrom, bpTo,
bpTo - bpFrom, len(bins), sum(bins.binIntegralNormed),
max(bins.binMaxCMS),
max(bins.binMaxCMS) / binsOverallMaxCMS,
np.mean(bins.binAvgCMS), min(bins.binRank),
max(bins.binRank))
assert numSegments > 0 and bpTotLen > 0 and gdTotLen > 0.0
intervalsStatsFile.writeRecord(replicaNum, numSegments, gdTotLen,
bpTotLen)
dbg('replicaNum numSegments gdTotLen bpTotLen')
def DefineRulesTo_fastCMS(pr,
pops,
chroms,
selPop,
sweepDir,
cmsDir,
genomeBuild='hg19'):
"""Define rules to do fast CMS computation.
Params:
pr - the PipeRun object to which to add rules
selPop - testing selection in which pop?
pops - comparing selPop to which pops?
sweepDir - the sweep directory
cmsDir - the directory under which CMS stats go
"""
pops = list(MakeSeq(pops))
if selPop not in pops: pops.append(selPop)
allPops = tuple(MakeSeq(pops))
if selPop not in allPops: allPops += (selPop, )
cmpPops = [pop for pop in allPops if pop != selPop]
rawScoresFN = {}
genMapSfx = genomeBuild2genMapSfx[genomeBuild]
for pop in allPops:
for chrom in chroms:
with pr.settingAttrs('pop chrom'):
snpInfoFN = os.path.join(
sweepDir,
'analysis/chr%(chrom)s/snps_%(pop)s.tsv' % locals())
projDir = os.path.join(sweepDir,
'data/chr%(chrom)s' % locals())
ancestralImportedFN = os.path.join(projDir,
'ancestral.tsv.imported')
genotypesImportedFN = os.path.join(
projDir,
'genotypes_chr%(chrom)s_%(pop)s_r21_nr_fwd_phased_all.imported'
% locals())
genMapImportedFN = os.path.join(
projDir,
'genetic_map_chr%(chrom)s_%(genMapSfx)s.txt.imported' %
locals())
pr.addRule(
name='extractSnpInfo',
commands=
'java -classpath ../Other/Ilya_Other/sweep/sweepsrc/sweep.jar edu.mit.broad.sweep.Main ExtractAlleleFreqs %(projDir)s/project %(snpInfoFN)s %(pop)s %(chrom)s'
% locals(),
commandsOld=
'java -classpath ../Other/Ilya_Other/sweep/sweepsrc/sweep/target/sweep-1.0-SNAPSHOT-jar-with-dependencies.jar edu.mit.broad.sweep.Main ExtractAlleleFreqs %(projDir)s/project %(snpInfoFN)s %(pop)s %(chrom)s'
% locals(),
depends_on=(ancestralImportedFN, genotypesImportedFN,
genMapImportedFN),
creates=snpInfoFN)
chr2dihhFN = {}
for chrom in chroms:
with pr.settingAttrs('chrom'):
chrom_s = 'chr' + str(chrom)
chromDir = os.path.join(cmsDir, chrom_s)
xpopScoresFN = os.path.join(
chromDir, AddFileSfx('max_xpop.tsv', chrom_s, selPop, pops))
pr.addInvokeRule(invokeFn=gatherXPOPscores,
invokeArgs=Dict('pops chrom selPop sweepDir',
outFN=xpopScoresFN),
attrs=dict(pop=allPops,
stat='max_xpop',
piperun_short=True))
ihsFN = getFN_ihs_signif(**Dict('sweepDir chrom', pop=selPop))
ihsScoresFN = os.path.join(
chromDir, AddFileSfx('iHS.tsv', chrom_s, selPop, pops))
dihhScoresFN = os.path.join(
chromDir, AddFileSfx('dihh.tsv', chrom_s, selPop, pops))
chr2dihhFN[chrom] = dihhScoresFN
pop2ancFreqFN = os.path.join(
cmsDir, chrom_s, AddFileSfx('pop2ancFreq.tsv', chrom_s, pops))
pop2sampleSizeFN = os.path.join(
cmsDir, chrom_s, AddFileSfx('pop2sampleSize.tsv', chrom_s,
pops))
pop2snpInfoFN = dict([(pop,
os.path.join(sweepDir, 'analysis',
'chr%(chrom)s' % locals(),
'snps_%(pop)s.tsv' % locals()))
for pop in pops])
pr.addInvokeRule(
invokeFn=gather_snp_info,
invokeArgs=Dict(
'pops pop2snpInfoFN pop2ancFreqFN pop2sampleSizeFN'))
pr.addInvokeRule(
invokeFn=gather_iHS_scores,
invokeArgs=Dict(
'chrom selPop ihsFN pop2ancFreqFN',
# snpInfoFN = pop2snpInfoFN[ selPop ],
ihsOutFN=ihsScoresFN,
dihhOutFN=dihhScoresFN),
attrs=dict(pop=selPop,
stat=('iHS', 'StdDiff'),
piperun_short=True))
freqDiffScoresFN = os.path.join(
chromDir, AddFileSfx('freqDiff.tsv', chrom_s, selPop, pops))
meanFstScoresFN = os.path.join(
chromDir, AddFileSfx('meanFst.tsv', chrom_s, selPop, pops))
pr.addInvokeRule(
invokeFn=computeMeanFstAndFreqDiffScores,
invokeArgs=Dict(
'chrom selPop sweepDir pops pop2ancFreqFN pop2sampleSizeFN',
outMeanFstFN=meanFstScoresFN,
outFreqDiffFN=freqDiffScoresFN),
attrs=dict(pop=allPops,
stat=('freqDiff', 'meanFst'),
piperun_short=True))
StdDiffScoresFN = os.path.join(
chromDir, AddFileSfx('StdDiff.tsv', chrom_s, selPop, pops))
rawScoresFN[chrom] = dict(iHS=ihsScoresFN,
StdDiff=StdDiffScoresFN,
meanFst=meanFstScoresFN,
freqDiff=freqDiffScoresFN,
max_xpop=xpopScoresFN)
# end: with pr.settingAttrs( 'chrom' )
# end: for chrom in chroms
# ihhStdFN = os.path.join( cmsDir, 'dihhstd.tsv' )
dihhGlobalStdFN = os.path.join(
cmsDir, AddFileSfx('dihh_global_std.tsv', selPop, pops))
dihhBinMeansFN = os.path.join(
cmsDir, AddFileSfx('dihh_bin_means.tsv', selPop, pops))
pr.addInvokeRule(invokeFn=normalizeByFreq_getMeanStd_tsv,
invokeArgs=dict(
iHHDiffFNs=[chr2dihhFN[k] for k in chroms],
globalStatFN=dihhGlobalStdFN,
binsStatFN=dihhBinMeansFN),
name='compute_dihh_meanstd')
# pr.addInvokeRule( invokeFn = computeMeanStd_binned_tsvs,
# invokeArgs = dict( inFNs = chr2dihhFN.values(), valCol = 'iHHDiff',
# binCol = 'normingFreqs', binMin = 0.05, binMax = 1.05, binStep = .05,
# outFN = ihhStdFN ),
# name = 'compute_dihh_std' )
for chrom in chroms:
with pr.settingAttrs('chrom'):
chrom_s = 'chr' + str(chrom)
chromDir = os.path.join(cmsDir, chrom_s)
StdDiffScoresFN = os.path.join(
chromDir, AddFileSfx('StdDiff.tsv', chrom_s, selPop, pops))
dbg('chrom chr2dihhFN[chrom]')
pr.addInvokeRule(invokeFn=normalizeByFreq_compute_normed_tsv,
invokeArgs=dict(iHHDiffFN=chr2dihhFN[chrom],
globalStatFN=dihhGlobalStdFN,
binsStatFN=dihhBinMeansFN,
StdDiffFN=StdDiffScoresFN))
statLikesRatioFNs = {}
for stat in CMSBins.CMSstats:
with pr.settingAttrs(
stat=stat,
pop=(selPop, ) if stat in ('iHS', 'StdDiff') else allPops,
piperun_short=True):
if stat not in CMSBins.nonNormedStats:
rawFNs = [rawScoresFN[chrom][stat] for chrom in chroms]
meanStdFN = os.path.join(
cmsDir, AddFileSfx('meanStd.tsv', stat, selPop, pops))
# DefineRulesTo_computeMeanStd( pr, inFNs = rawFNs, colNum = 1,
# outFN = meanStdFN,
# addRuleArgs = \
# dict( name = 'computeMeanStd_for_stat',
# attrs = dict( chrom = chroms ) ) )
# meanStdBzFN = os.path.join( cmsDir, stat + '_meanStdForStat.tsv' )
pr.addInvokeRule(invokeFn=computeMeanStd,
invokeArgs=dict(inFNs=rawFNs,
colName=stat,
outFN=meanStdFN))
# end: if stat not in CMSBins.nonNormedStats
for chrom in chroms:
with pr.settingAttrs('chrom'):
statFN = rawScoresFN[chrom][stat]
if stat not in CMSBins.nonNormedStats:
normedFN = AddFileSfx(statFN, 'normed')
DefineRulesTo_normalizeOneColumn(
pr,
inFN=statFN,
meanStdFN=meanStdFN,
colName=stat,
outFN=normedFN,
addRuleArgs=dict(attrs=Dict('chrom')))
statFN = normedFN
bins_beg = CMSBins.stat_start[stat]
bins_end = CMSBins.stat_end[stat]
bins_n = CMSBins.stat_nbin[stat]
statLikesRatioFN = AddFileSfx(rawScoresFN[chrom][stat],
'likesRatio')
statLikesRatioFNs[(chrom, stat)] = statLikesRatioFN
pr.addInvokeRule(
invokeFn=computeLikeRatioForStat,
invokeArgs=dict(
stat=stat,
statValsFN=statFN,
hitLikesFN=
'../Data/Common_Data/sim/likes/hitsLikes_toneutFixed_1.tsv',
missLikesFN=
'../Data/Common_Data/sim/likes/missLikes_toneutFixed_1.tsv',
stat_start=bins_beg,
stat_end=bins_end,
stat_nbin=bins_n,
statLikesRatioFN=statLikesRatioFN))
# end: with pr.settingAttrs( 'chrom' )
# end: for chrom in chroms
# end: with pr.settingAttrs( stat = stat, piperun_short = True )
# end: for stat in CMSBins.CMSstats
for chrom in chroms:
with pr.settingAttrs(chrom=chrom, stat=CMSBins.CMSstats):
chrom_s = 'chr' + str(chrom)
chromDir = os.path.join(cmsDir, chrom_s)
likesRatioFN = os.path.join(
chromDir,
AddFileSfx('likesRatio.tsv', CMSBins.CMSstats, selPop, pops))
pr.addInvokeRule(invokeFn=addLikesRatios,
invokeArgs=dict(
inFNs=[
statLikesRatioFNs[(chrom, stat)]
for stat in CMSBins.CMSstats
],
colNames=[
colName + 'likeRatio'
for colName in CMSBins.CMSstats
],
outFN=likesRatioFN))
def RunTasks( options ):
"""Take tasks from the specified queue directory, and run them.
Params:
options - see command-line parameter definition in main() below.
"""
if haveParamiko: Random.atfork()
startClock = time.time()
logging.info( 'Starting runner (process id %d on host %s) with options %s'
% ( os.getpid(), GetHostName(), options ) )
stopSignal = [ False ]
def SetStopSignal( sigNum, stkFrm ):
logging.info( 'Setting stop signal to stop runners' )
stopSignal[ 0 ] = True
dbg( '"aftset" stopSignal' )
signal.signal( signal.SIGUSR1, SetStopSignal )
fs = RemoteFileSystem( remote = options.remote, pw = options.password, pkey = options.pkey ) \
if options.remote else LocalFileSystem()
# check that all queues exist
assert all( fs.exists( queue ) and fs.isdir( queue )
for queue in options.queues.split( fs.pathsep ) )
# register a cleanup routine so that, if we claim a task and then
# crash midway through it, our lock on the task is erased, so that
# another runner can pick up the task. Note that if the _task_ fails
# with an error code, that's fine -- we just report the error code
# to the mqsub.sh script instance that submitted the task.
# The cleanup here happens only if the runner crashes before receiving
# a proper exit code from the task.
fileToErase = [ None ]
@atexit.register
def DoErase( eraseWhat = fileToErase ):
if eraseWhat[0] and fs.exists(eraseWhat[0]):
#print 'runner AtExit: removing ' + eraseWhat[0]
fs.remove( eraseWhat[0] )
# var: lastFinish - time when a task last finished.
lastFinish = time.time()
queues = options.queues.split( fs.pathsep )
lastQueueModTime = [ None ] * len( queues )
skipDirs = set([ 'newtask.dat' ] )
numTasksRun = 0
numProcsAvail = int( os.getenv( 'LSB_DJOB_NUMPROC', 1 ) )
dbg( 'numProcsAvail' )
for queue in queues:
EnsureDirExists( os.path.join( queue, 'succ' ) )
EnsureDirExists( os.path.join( queue, 'fail' ) )
while not stopSignal[0]:
ranCommand = False
if options.maxRunHours > 0 and ( time.time() - startClock ) / 3600.0 > options.maxRunHours:
logging.info( 'Runner exiting after CPU time of %s hours' % ( time.time() - startClock ) / 3600.0 )
return
if stopSignal[ 0 ]:
logging.info( 'Runner stopped by stop signal' )
return
else: dbg( '"chkstop" stopSignal' )
dbg( 'queues' )
for queueNum, queue in enumerate( queues ):
dbg( 'queueNum queue' )
# do a quick check to see if any tasks have been added to the queue since we last checked
newTaskFN = os.path.join( queue, 'newtask.dat' )
try:
curQueueModTime = fs.stat( newTaskFN ).st_mtime
if curQueueModTime == lastQueueModTime[ queueNum ]: continue
lastQueueModTime[ queueNum ] = curQueueModTime
except EnvironmentError as e:
if os.path.exists( newTaskFN ):
logging.warning( 'ERROR CHECKING FOR NEW TASKS in queue %s: %s' % ( queue, e ) )
pass
# find an unclaimed task in this queue, and try to claim it
taskDirs = sorted( fs.listdir( queue ) )
dbg( 'len(taskDirs)' )
#random.shuffle( taskDirs )
dbg( 'os.environ.get("MQ_FIRST_DIR")' )
if 'MQ_FIRST_DIR' in os.environ and os.environ[ 'MQ_FIRST_DIR' ] in taskDirs:
taskDirs = [ os.environ[ 'MQ_FIRST_DIR' ] ] + taskDirs
logging.info( 'putting specified dir first' )
for taskDir in taskDirs:
if taskDir in skipDirs: continue
if options.maxRunHours > 0 and ( ( time.time() - startClock ) / 3600.0 ) > options.maxRunHours:
logging.info( 'Runner exiting after CPU time of %s hours' % ( ( time.time() - startClock ) / 3600.0 ) )
return
if stopSignal[ 0 ]:
logging.info( 'Runner stopped by stop signal' )
return
else: dbg( '"chkstop" stopSignal' )
try:
while fs.path.exists( os.path.join( queue, 'noclaim.dat' ) ):
time.sleep( 60 + random.normalvariate( 10.0, 5.0 ) )
fullTaskDir = fs.path.join( queue, taskDir )
claimedFN = fs.path.join( fullTaskDir, options.claimedFN )
attrsFN = fs.path.join( fullTaskDir, 'attrs.tsv' )
cwdFN = fs.path.join( fullTaskDir, 'submitdir.txt' )
failedCond = []
def saveVal( name, val, fc = failedCond ):
if not val: fc.append( name )
return val
if saveVal( 'ready', fs.path.exists( fs.path.join( fullTaskDir, options.readyFN ) ) ) \
and saveVal( 'not claimed', not fs.path.exists( claimedFN ) ) \
and saveVal( 'relocatable', ( not options.remote or \
all([ not f.startswith( '/' ) for which in ( 'sources', 'targets' ) \
for f in fs.SlurpFile( fs.path.join( fullTaskDir, which + '.lst' ) ) ]) ) ) \
and saveVal( 'memOk', GetMemReq( fs, attrsFN ) <= options.maxMem ) \
and saveVal( 'minMemOk', options.minMem == 0 or GetMemReq( fs, attrsFN ) >= options.minMem ) \
and saveVal( 'minProc', GetProcReq( fs, attrsFN ) >= options.minProc ) \
and saveVal( 'maxProc', GetProcReq( fs, attrsFN ) <= numProcsAvail ) \
and saveVal( 'local', ( options.local_tasks or not GetTaskAttr( fs, attrsFN, 'piperun_run_locally', False ) ) ) \
and saveVal( 'onlyLocal',
( not options.only_local_tasks or GetTaskAttr( fs, attrsFN, 'piperun_run_locally', False ) ) ) \
and saveVal( 'short', ( not options.runOnlyShort or GetTaskAttr( fs, attrsFN, 'piperun_short', False ) ) ) \
and saveVal( 'long', ( not options.runOnlyLong or not GetTaskAttr( fs, attrsFN, 'piperun_short', False ) ) ) \
and saveVal( 'notRequeued', ( not options.noRequeuedTasks or not fs.path.exists( fs.path.join( fullTaskDir, 'requeued.dat' ) ) ) ) \
and saveVal( 'notFromHost', ( not options.onlyFromHost or socket.getfqdn() == options.onlyFromHost ) ) \
and saveVal( 'notFromPipeline', ( not options.onlyFromPipelineId or \
GetTaskAttr( fs, attrsFN, 'piperun_pipelineId' ) == options.onlyFromPipelineId ) ):
# try to claim the task
try:
fd = fs.open(claimedFN, os.O_CREAT|os.O_EXCL|os.O_WRONLY)
except EnvironmentError:
# another runner beat us to this task -- go and check other tasks
logging.info( 'another job beat us to claiming ' + fullTaskDir )
continue
try:
fs.write( fd, 'locked by process %d on host %s\n' % ( os.getpid(), GetHostName() ) )
for v in list(os.environ.keys()):
fs.write( fd, '%s=%s\n' % ( v, os.environ[v] ) )
finally:
fs.close( fd )
# Tell our cleanup code to release this task if we crash.
fileToErase[0] = claimedFN
# get the command to run the task
theCMD = fs.SlurpFile( os.path.join( fullTaskDir, 'command.dat' ) ).strip()
theCmdDir = fs.SlurpFile( os.path.join( fullTaskDir, 'submitdir.txt' ) ).strip()
theCmdEnvFN = os.path.join( fullTaskDir, 'submitenv.txt' )
if options.remote:
assert have_fcntl
SystemSucceed( 'mkdir -p ' + os.path.join( options.localDataDir, fs.root[1:] ) )
for needDir in 'Operations', 'Classes', 'System', 'Other':
needDirFull = os.path.join( options.localDataDir, fs.root[1:], '..', needDir )
if not os.path.exists( needDirFull ):
os.symlink( os.path.realpath( os.path.join( '..', needDir ) ), needDirFull )
# copy source files
# get exclusive locks on the source files
srcFiles = sorted( set( fs.SlurpFile( os.path.join( fs.root, fullTaskDir, 'sources.lst' ) ).rstrip( '\n' ).split( '\n' ) ) )
srcLockIds = []
srcLockFiles = []
for srcFile in srcFiles:
lockFile = os.path.join( options.localDataDir, 'mqlocks', srcFile[1:] )
if lockFile.endswith('/'): lockFile = lockFile[:-1]
lockFile += '.lock'
SystemSucceed( 'mkdir -p ' + os.path.dirname( lockFile ) )
gotLock = False
while not gotLock:
try:
openMode = os.O_CREAT|os.O_EXCL|os.O_WRONLY
logging.info( 'opening ' + lockFile + ' with mode ' + str( openMode ) )
lockId = os.open( lockFile, openMode )
gotLock = True
except EnvironmentError:
logging.info( 'Could not create ' + lockFile + ' , waiting...' )
time.sleep( 10 + random.normalvariate( 3.0, 1.0 ) )
fcntl.lockf( lockId, fcntl.LOCK_EX )
srcLockIds.append( lockId )
srcLockFiles.append( lockFile )
logging.info( 'Got lock on ' + lockFile )
SystemSucceed( 'rsync -zprv --files-from=:' + os.path.join( fs.root, fullTaskDir, 'sources.lst' ) +
' ' + fs.username + '@' + fs.hostname + ':/ ' + options.localDataDir )
for srcLockId, srcLockFile in zip( srcLockIds, srcLockFiles)[::-1]:
fcntl.lockf( srcLockId, fcntl.LOCK_UN )
os.close( srcLockId )
SystemSucceed( 'rm -rf ' + srcLockFile )
targets = fs.SlurpFile( os.path.join( fs.root, fullTaskDir, 'targets.lst' ) ).rstrip( '\n' ).split('\n')
targetDirs = set( map( os.path.dirname, [_f for _f in map( str.strip, targets ) if _f] ) )
dbg( '"DDDDDD" targetDirs' )
for targetDir in targetDirs:
assert targetDir.startswith( '/' )
tdir = os.path.join( options.localDataDir, targetDir[1:] )
SystemSucceed( 'mkdir -p ' + tdir + ' ' + os.path.join( tdir, 'makeinfo' ) )
theCMD = 'cd ' + os.path.join( options.localDataDir, fs.root[1:] ) + ' && ' + theCMD
logging.info( 'Under ' + claimedFN + ' RUNNING: ' + theCMD )
# Actually run the task; get its exit code
save_cwd = os.getcwd()
try:
os.chdir( theCmdDir )
logging.info( 'CWD=' + os.getcwd() )
runScriptFN = os.path.join( fullTaskDir, 'run.sh' )
with open( runScriptFN, 'w' ) as out:
out.write( '#!/usr/bin/env bash\n' )
out.write( 'set -e -o pipefail\n' )
with open( theCmdEnvFN ) as envFile:
for line in envFile:
if '=' not in line or line.startswith('module='): break
equalIdx = line.index( '=' )
envVarName = line[ :equalIdx+1 ]
if not ( re.search( r'\W', envVarName ) or envVarName.startswith( 'LSB_' ) or \
envVarName.startswith( 'LSF_' ) or \
envVarName.startswith( 'LS_' ) or envVarName.startswith( 'SLURM' ) or \
envVarName in \
( 'SYS_TYPE', 'MACHTYPE', 'VENDOR', 'OSTYPE',
'DOMAINNAME', 'HOSTTYPE', 'SHORTHOST', 'SSH_TTY',
'HOST', 'HOSTNAME', 'REMOTEHOST', 'STY' ) ):
out.write( 'export ' + envVarName + "'" + line[ equalIdx+1: -1 ] + "'\n" )
out.write( theCMD )
os.chmod( runScriptFN, stat.S_IXUSR | stat.S_IRWXU )
try:
exitCode = os.system( runScriptFN )
except ( KeyboardInterrupt, SystemExit ):
interruptedFN = os.path.join( fullTaskDir, 'interrupted.dat' )
DumpFile( interruptedFN, 'interrupted' );
raise
finally:
os.chdir( save_cwd )
logging.info( 'Under ' + claimedFN + ' FINISHED RUNNING: ' + theCMD )
logging.info( 'Got exit code %d' % exitCode )
if options.remote:
# copy the target files and the output log back to the correct dirs on the remote system
# first, make sure the files all exist, and are no longer being written to.
time.sleep( options.aftTaskDelay )
os.system( 'rsync -zprv --files-from=:' + os.path.join( fs.root, fullTaskDir, 'targets.lst' ) +
' ' + options.localDataDir + ' ' + fs.username + '@' + fs.hostname + ':/' )
# If we succeeded in running the task (whether the task itself failed or not),
# tell the cleanup code to NOT release this task if we crash.
fileToErase[0] = None
# Tell the task submitter script that we are done, and what the task's
# exit code was.
if os.path.exists( os.path.join( fullTaskDir, 'nmq.dat' ) ):
time.sleep(3)
fd = fs.open( os.path.join( fullTaskDir, 'completed.dat' ), os.O_CREAT|os.O_EXCL|os.O_WRONLY )
fs.close(fd)
try:
shutil.move( fullTaskDir, os.path.join( queue, 'succ' if exitCode == 0 else 'fail' ) )
except EnvironmentError as e:
logging.warning( 'Error moving ' + fullTaskDir + ' to ' + os.path.join( queue, 'succ' if exitCode == 0 else 'fail' ) + ' : ' + e )
else:
exitCodeFN = os.path.join( fullTaskDir, 'exitCode.dat' )
fd = fs.open( exitCodeFN, os.O_CREAT|os.O_EXCL|os.O_WRONLY )
bytesWritten = fs.write( fd, str( exitCode ) )
fs.close( fd )
time.sleep(3)
logging.info( 'Wrote exit code %s to file %s (%s bytes)' % ( exitCode, exitCodeFN, bytesWritten ) )
fd = fs.open( os.path.join( fullTaskDir, 'completed.dat' ), os.O_CREAT|os.O_EXCL|os.O_WRONLY )
fs.close(fd)
# Record that we actually ran a task here.
ranCommand = True
lastFinish = time.time()
numTasksRun += 1
else:
logging.info( 'did not take task ' + taskDir + ' ; reason: ' + str( failedCond ) );
except:
excInfo = sys.exc_info()
logging.warning( 'Error trying to grab task from ' + taskDir + ' (%s), skipping...'
% str( excInfo ) )
traceback.print_exc()
dbg( 'ranCommand lastFinish time.time()-lastFinish' )
if not ranCommand:
waitTimeHere = time.time() - lastFinish
if ( numTasksRun > 0 and options.maxWaitTime > 0 and waitTimeHere > options.maxWaitTime ) \
or ( numTasksRun == 0 and options.maxFirstWaitTime > 0 and waitTimeHere > options.maxFirstWaitTime ) :
logging.info( 'Runner exiting after idle time of %s' % waitTimeHere )
return
time.sleep( options.taskCheckInterval + random.normalvariate( 3.0, 1.0 ) )
def computeMeanStd_binned(inDatas, valCol, binCol, binMin, binMax, binStep):
"""Compute binned stats for a set of tables"""
binCount = int((binMax - binMin) / binStep)
dbg('binCount')
sums = np.zeros(binCount)
sumsSq = np.zeros_like(sums)
counts = np.zeros_like(sums)
bins = np.arange(binMin, binMax, binStep)
for d_idx, d in enumerate(inDatas):
dbg('d_idx d binStep')
dbg('d[binCol]')
binColValues = 1.0 - (1.0 - d[binCol].values)
for i in range(binCount):
# binBot = bins[i]
binTop = bins[i]
theIdx = ((binTop - binColValues) < binStep) & (
(binTop - binColValues) > 0)
# theIdx = ( binBot < d[ binCol ].values ) & ( d[ binCol ].values <= binTop )
# DotData( names = ('rows',), Columns = theIdx.nonzero() ).saveToSV( 'nz%02d.tsv' % i )
#rowsStr = ','.join(map(str,list(theIdx.nonzero())))
#print 'binnedRows=', rowsStr
hereVals = d[theIdx][valCol]
# DotData( names = ( 'temp', ), Columns = ( hereVals, ) ).saveToSV( 'temp2%2d.tsv' % i )
dbg('"BEF" theIdx.sum() i bins[i] len(hereVals)')
counts[i] += len(hereVals)
sums[i] += np.sum(hereVals)
sumsSq[i] += np.sum(hereVals * hereVals)
# dbg( '"AFT" i bins[i] bins[i+1] len(hereVals)' )
if False:
# fast version
binsHere = np.digitize(d[binCol], bins) - 1
dbg('len(binsHere) binsHere')
np.clip(binsHere, 0, binCount - 1, out=binsHere)
dbg('binsHere')
counts += np.bincount(binsHere, minlength=binCount)
sums += np.bincount(binsHere,
weights=d[valCol],
minlength=binCount)
sumsSq += np.bincount(binsHere,
weights=d[valCol] * d[valCol],
minlength=binCount)
countsOrig = counts.astype(int)
counts[counts == 0] = np.nan
means = sums / counts
stds = sumsSq / counts - means * means
return pd.DataFrame(
dict(binBeg=bins - binStep,
binEnd=bins,
counts=countsOrig,
sums=sums,
sumsSq=sumsSq,
means=means,
stds=stds))
def PushResults( options ):
"""Push finished results"""
logging.info( 'Starting result pusher as process %d on host %s with options %s' %
( os.getpid(), GetHostName(), options ) )
# check that all queues exist
assert all( os.path.exists( queue ) and os.path.isdir( queue )
for queue in options.queues.split( os.pathsep ) )
stopSignal = [ False ]
def SetStopSignal( sigNum, stkFrm ):
logging.info( 'Setting stop signal to stop pushers' )
stopSignal[ 0 ] = True
dbg( '"aftset" stopSignal' )
signal.signal( signal.SIGUSR2, SetStopSignal )
while not stopSignal[0]:
for queue in options.queues.split( os.pathsep ):
logging.info( 'Pushing results in ' + queue + '...' )
# find an unclaimed task in this queue, and try to claim it
try: taskDirs = filter( lambda f: f.startswith('mq'), os.listdir( queue ) )
except EnvironmentError as e:
logging.info( 'Error getting list of tasks in queue ' + queue + ': ' + str( e ) )
# sleep a bit -- maybe it's some transient condition that will resolve itself
time.sleep( 60 + random.normalvariate( 3.0, 1.0 ) )
continue
for taskDir in taskDirs:
fullTaskDir = os.path.join( queue, taskDir )
try:
pushingFN = os.path.join( fullTaskDir, 'pushing.dat' )
submithostFN = os.path.join( fullTaskDir, 'submithost.txt' )
if os.path.exists( os.path.join( fullTaskDir, 'completed.dat' ) ) \
and os.path.exists( submithostFN ) and GetSubmitHost( submithostFN ) == GetHostName() \
and not os.path.exists( pushingFN ):
try:
fd = os.open( pushingFN, os.O_CREAT|os.O_EXCL|os.O_WRONLY )
except EnvironmentError as e:
if e.errno not in ( errno.EEXIST, errno.EACCES, errno.EAGAIN ): raise
# another resultPusher beat us to this task -- go and check other tasks
continue
os.write( fd, 'result being pushed by process %d on host %s' % ( os.getpid(), GetHostName() ) )
os.close( fd )
taskDescr = ''
try:
attrsFN = os.path.join( fullTaskDir, 'attrs.tsv' )
if os.path.exists( attrsFN ):
taskDescr += ' output in ' + GetTaskAttr( attrsFN, 'piperun_outputSavedTo' )
except EnvironmentError as e:
logging.info( 'Could not read attrs for task in ' + fullTaskDir + ': ' + str( e ) )
try:
infoFNs = [ os.path.join( fullTaskDir, f ) for f in 'command.dat', 'attrs.tsv', 'claimed.dat' ]
infoContents = '\n'.join([ SlurpFile( f ) if os.path.exists( f ) else 'missing file: ' + f
for f in infoFNs ])
thePipe = os.open( os.path.join( fullTaskDir, 'getresult' ), os.O_WRONLY | os.O_NONBLOCK )
exitCodeReadOk = False
writeOk = False
closeOk = False
exitCode = 'UNKNOWN'
try:
exitCode = SlurpFile( os.path.join( fullTaskDir, 'exitCode.dat' ) ).strip()
exitCodeReadOk = True
os.write( thePipe, exitCode )
writeOk = True
finally:
os.close( thePipe )
closeOk = True
logging.info( 'Pushed result in ' + fullTaskDir + ': ' + ( 'nonzero ' if exitCode != '0' else '' ) + 'exit code ' + exitCode + taskDescr )
if not writeOk or not closeOk or not exitCodeReadOk: dbg( 'exitCodeReadOk writeOk closeOk' )
if exitCodeReadOk and exitCode != '0': logging.info( infoContents )
except EnvironmentError as e:
logging.info( 'The task at ' + fullTaskDir + ' seems to have been orphaned: ' + e.strerror )
except EnvironmentError as e:
logging.info( 'Error processing queue ' + queue + ' task ' + taskDir + ': ' + str( e ) )
# sleep a bit -- maybe it's some transient condition that will resolve itself
time.sleep( 60 + random.normalvariate( 3.0, 1.0 ) )
# if we pushed at least something, go back and try again. if not, wait.
time.sleep( options.sleepInterval + random.normalvariate( 3.0, 1.0 ) )
default=1000,
help='number of boostrap iters')
parser.add_argument('--cosi-binary',
default='./coalescent',
help='cosi binary to run')
print 'calling parser'
args = parser.parse_args()
print 'parser done'
# do a reference run
print 'generating reference'
SystemSucceed(' '.join(
map(str, (args.cosi_binary, '-p', '1_simple.cosiParams', '-n', 100,
'-m'))) + ' | sample_stats_extra > ref.tsv')
refData = DotData(SVPath='ref.tsv')
min_p = np.ones(len(refData.dtype.names))
max_D = np.repeat(-np.inf, len(refData.dtype.names))
for i in range(10):
dbg('i')
refFN = 'reftest%d.tsv' % i
SystemSucceed(' '.join(
map(str, (args.cosi_binary, '-p', '0_simple.cosiParams', '-n', 100,
'-m'))) + ' | sample_stats_extra > ' + refFN)
z = DotData(SVPath=refFN)
for colNum, col in enumerate(z.dtype.names):
ks_D, ks_p = stats.ks_2samp(refData[col], z[col])
min_p[colNum] = np.min((min_p[colNum], ks_p))
max_D[colNum] = np.max((max_D[colNum], ks_D))
dbg('i min_p max_D')
def SetStopSignal( sigNum, stkFrm ):
logging.info( 'Setting stop signal to stop pushers' )
stopSignal[ 0 ] = True
dbg( '"aftset" stopSignal' )
