#sample = np.random.uniform(0, 1, nStates)
#samples = hmc.run(0, 2000, sample)
#avgWeights = np.sum(samples, axis=0) / samples.shape[0]
#initialWeights = avgWeights
#stationaryDistEst = np.exp(avgWeights) / np.sum(np.exp(avgWeights))
# update stationary distribution elements to the latest value
#initialStationaryDist = stationaryDistEst
# sample exchangeable coefficients using local bouncy particle sampler
## define the model
model = ExpectedCompleteReversibleModelWithBinaryFactors(expectedCompleteReversibleObjective, nStates, initialBinaryWeights, stationaryDist , bivariateFeatIndexDictionary)
## define the sampler to use
## local sampler to use
allFactors = model.localFactors
localSampler = LocalRFSamplerForBinaryWeights(model, rfOptions, mcmcOptions, nStates, bivariateFeatIndexDictionary)
phyloLocalRFMove = PhyloLocalRFMove(model, localSampler, initialBinaryWeights)
initialBinaryWeights = phyloLocalRFMove.execute()
print("The initial estimates of the binary weights are:")
print(initialBinaryWeights)
initialRateMtx = ReversibleRateMtxPiAndBinaryWeightsWithGraphicalStructure(nStates, stationaryWeights, initialBinaryWeights,
bivariateFeatIndexDictionary)
#initialStationaryDist = initialRateMtx.getStationaryDist()
initialRateMatrix = initialRateMtx.getRateMtx()
initialExchangeCoef = initialRateMtx.getExchangeCoef()
print("The initial estimates of the exchangeable parameters are:")
print(initialExchangeCoef)
print(initialRateMatrix)
#print(initialStationaryDist)
def run(self, initialWeights=None):
## for every batchSize number of samples, we get the total sum
## of these samples and then refresh it to zero after we reach the
## batch size
## we only check the stationary distribution and exchangeable parameters
weightsDistBatchSum = np.zeros((1, self.dim))
initialWeights = self.generateInitialSamples(
initialWeightsDist="Normal")
# create arrays to save the posterior samples
postSamples = self.generateArraysToSaveSamples()
if self.dumpResultIteratively:
allFileNames = self.createAllOutputFileNames(
self.dir_name,
self.nMCMCIter,
trajectoryLength=self.trajectoryLength,
mcmcSeed=self.initialSampleSeed)
# this algorithm runs a combination of HMC and local BPS
startTime = datetime.now()
previousWeightsMean = np.zeros((1, self.dim))
for i in range(self.nMCMCIter):
postSamples[i, :] = initialWeights
weightsDistBatchSum = weightsDistBatchSum + initialWeights
if i > 0 and (i + 1) % self.batchSize == 0:
## When we reach the batch size, refresh the sum
## write the currrent file to csv and then refresh the vector to zeros
weightsDistBatchMean = self.getErgodicMean(
nSamples=int(i + 1),
previousMean=previousWeightsMean,
newPartialSum=weightsDistBatchSum,
batchSize=self.batchSize)
previousWeightsMean = weightsDistBatchMean
self.dumpResult(weightsDistBatchMean[0, :],
allFileNames['weightsErgodicMean'])
weightsDistBatchSum = np.zeros(self.dim)
if i > 0 and (i + 1) % self.dumpResultIterations == 0:
## record the results
self.dumpResult(
postSamples[(i + 1 - self.dumpResultIterations):(i +
1), :],
allFileNames['weights'])
model = ExpectedCompleteReversibleModelBinaryFactors.ModelWithNormalFactors(
initialWeights)
if i == 0:
self.neighborVariablesForAllFactors = neighborVariableForAllFactors(
model.localFactors)
self.variableAndFactorInfo = neighbourVariblesAndFactorsAndExtendedNeighborsOfAllFactorsDict(
model.localFactors)
self.indexOfFactorsForEachBivariateFeat = getIndexOfNeighborFactorsForEachIndexOfBinaryFeature(
self.dim)
localSampler = LocalRFSamplerForBinaryWeights(
model, self.rfOptions, self.mcmcOptions, 0,
self.neighborVariablesForAllFactors,
self.variableAndFactorInfo,
self.indexOfFactorsForEachBivariateFeat)
phyloLocalRFMove = PhyloLocalRFMove(model=model,
sampler=localSampler,
initialPoints=initialWeights,
options=self.rfOptions,
prng=RandomState(i))
initialWeights = phyloLocalRFMove.execute()
print("The current iteration finished is:")
print(i)
endTime = datetime.now()
timeElapsed = (endTime - startTime).total_seconds()
if not self.dumpResultIteratively:
result = {}
result['postSamples'] = postSamples
result['elapsingTime'] = timeElapsed
return result
else:
self.outputRunningTime(actualRunningTimeInSec=timeElapsed,
dir_name=self.dir_name,
time_base_filename="wallTime",
nMCMCIter=self.nMCMCIter)
def testLocalBPSForStationaryAndBivariateWeights(data, nMCMCIter = 500, trajectoryLength=0.1):
avgNTrans = data['transitCount']
avgHoldTimes = data['sojourn']
avgNInit = data['nInit']
bivariateDictionary = data['bivariateDictionary']
prng = np.random.RandomState(1)
nStates = len(data['stationaryWeights'])
## run HMC to estimate the stationary distribution
initialExchangeCoef = np.exp(prng.uniform(0, 1, int(nStates *(nStates-1)/2)))
## construct expected complete reversible model objective
expectedCompleteReversibleObjective = ExpectedCompleteReversibleObjective(avgHoldTimes, avgNInit, avgNTrans, 1.0, initialExchangeCoef)
initialWeights = data['stationaryWeights']
stationaryDistEst = np.exp(initialWeights) / np.sum(np.exp(initialWeights))
# update stationary distribution elements to the latest value
initialStationaryDist = stationaryDistEst
# sample exchangeable coefficients using local bouncy particle sampler
## define the model
if isinstance(initialExchangeCoef, float):
initialExchangeCoef = [initialExchangeCoef]
model = ExpectedCompleteReversibleModelWithBinaryFactors(expectedCompleteReversibleObjective, nStates,
np.log(initialExchangeCoef),
initialStationaryDist,
bivariateDictionary)
## define the sampler to use
## local sampler to use
rfOptions = RFSamplerOptions(trajectoryLength=trajectoryLength)
mcmcOptions = MCMCOptions(nMCMCIter,1,0)
nBivariateFeat = int(nStates *(nStates-1)/2)
initialBinaryWeights = prng.normal(0, 1, nBivariateFeat)
seed = 3
stationarySamples = np.zeros((nMCMCIter, nStates))
binaryWeightsSamples = np.zeros((nMCMCIter, nBivariateFeat))
exchangeableSamples = np.zeros((nMCMCIter, len(initialExchangeCoef)))
####### below is the older version of the sampler
for i in range(nMCMCIter):
# save the samples of the parameters
stationarySamples[i, :] = initialStationaryDist
binaryWeightsSamples[i, :] = initialBinaryWeights
exchangeableSamples[i, :] = initialExchangeCoef
localSampler = LocalRFSamplerForBinaryWeights(model, rfOptions, mcmcOptions, nStates, bivariateDictionary)
phyloLocalRFMove = PhyloLocalRFMove(model, localSampler, initialBinaryWeights, options=rfOptions, randomSeed=i)
initialBinaryWeights = phyloLocalRFMove.execute()
print("The initial estimates of the exchangeable param are:")
print(initialExchangeCoef)
# print(initialBinaryWeights)
# localSamplerOld = LocalRFSamplerForBinaryWeightsOldVersion(model, rfOptions, mcmcOptions, nStates,
# bivariateFeatIndexDictionary)
# phyloLocalRFMove = PhyloLocalRFMove(seed, model, localSamplerOld, initialBinaryWeights)
# initialBinaryWeightsOld = phyloLocalRFMove.execute()
initialRateMtx = ReversibleRateMtxPiAndBinaryWeightsWithGraphicalStructure(nStates, initialWeights,
initialBinaryWeights,
bivariateDictionary)
initialStationaryDist = initialRateMtx.getStationaryDist()
initialExchangeCoef = initialRateMtx.getExchangeCoef()
print(i)
result = {}
result['stationaryDist'] = stationarySamples
result['binaryWeights'] = binaryWeightsSamples
result['exchangeableCoef'] = exchangeableSamples
return result
def succesiveConditionalSimulatorWithInitialWeightAndData(
self, initialParamAndData, K, nStates, nBivariateFeat,
bivariateFeatDictionary, seed, bt, nSeq, interLength, HMCPlusBPS,
onlyHMC, nLeapFrogSteps, stepSize, nItersPerPathAuxVar, rfOptions,
mcmcOptions):
thetaStationaryWeights = initialParamAndData['stationaryWeights']
thetaBinaryWeights = initialParamAndData['binaryWeights']
theta0StationarySamples = np.zeros((K, nStates))
theta0BinaryWeightsSamples = np.zeros((K, nBivariateFeat))
## for debuging reasons, we ave the stationary distribution and exchangeable coef
## actually, we only need the last sample at the last iteration K
theta0StationaryDistSamples = np.zeros((K, nStates))
theta0ExchangeableCoefSamples = np.zeros(
(K, int(nStates * (nStates - 1) / 2)))
theta0StationarySamples[0, :] = thetaStationaryWeights
theta0BinaryWeightsSamples[0, :] = thetaBinaryWeights
theta0StationaryDistSamples[
0, :] = np.exp(thetaStationaryWeights) / np.sum(
np.exp(thetaStationaryWeights))
initialExchangeCoef = getExchangeCoef(nStates, thetaBinaryWeights,
bivariateFeatDictionary)
theta0ExchangeableCoefSamples[0, :] = initialExchangeCoef
sample = None
if onlyHMC:
sample = np.zeros((nStates + nBivariateFeat))
sample[0:nStates] = thetaStationaryWeights
sample[nStates:(nStates + nBivariateFeat)] = thetaBinaryWeights
if HMCPlusBPS:
sample = thetaStationaryWeights
suffStat = initialParamAndData['suffStat']
nInit = suffStat['nInit']
nTrans = suffStat['nTrans']
holdTime = suffStat['sojourn']
expectedCompleteReversibleObjective = None
for i in np.arange(1, K, 1):
if HMCPlusBPS:
expectedCompleteReversibleObjective = ExpectedCompleteReversibleObjective(
holdTime, nInit, nTrans, 1.0, initialExchangeCoef)
if onlyHMC:
expectedCompleteReversibleObjective = ExpectedCompleteReversibleObjective(
holdTime,
nInit,
nTrans,
1.0,
nBivariateFeatWeightsDictionary=bivariateFeatDictionary)
#####################################
hmc = HMC(nLeapFrogSteps, stepSize,
expectedCompleteReversibleObjective,
expectedCompleteReversibleObjective)
lastSample = sample
for k in range(nItersPerPathAuxVar):
hmcResult = hmc.doIter(nLeapFrogSteps, stepSize, lastSample,
expectedCompleteReversibleObjective,
expectedCompleteReversibleObjective,
True)
lastSample = hmcResult.next_q
sample = lastSample
if onlyHMC:
thetaStationaryWeights = sample[0:nStates]
thetaBinaryWeights = sample[nStates:(nStates + nBivariateFeat)]
theta0StationarySamples[i, :] = thetaStationaryWeights
theta0BinaryWeightsSamples[i, :] = thetaBinaryWeights
if HMCPlusBPS:
thetaStationaryWeights = sample
theta0StationarySamples[i, :] = thetaStationaryWeights
# update stationary distribution elements to the latest value
thetaStationaryDist = np.exp(sample) / np.sum(np.exp(sample))
# sample exchangeable coefficients using local bouncy particle sampler
## define the model
model = ExpectedCompleteReversibleModelWithBinaryFactors(
expectedCompleteReversibleObjective, nStates,
thetaBinaryWeights, thetaStationaryDist,
bivariateFeatDictionary)
## define the sampler to use
## local sampler to use
localSampler = LocalRFSamplerForBinaryWeights(
model, rfOptions, mcmcOptions, nStates,
bivariateFeatDictionary)
phyloLocalRFMove = PhyloLocalRFMove(
model=model,
sampler=localSampler,
initialPoints=thetaBinaryWeights,
options=rfOptions,
prng=RandomState(i))
thetaBinaryWeights = phyloLocalRFMove.execute()
theta0BinaryWeightsSamples[i, :] = thetaBinaryWeights
initialRateMtx = ReversibleRateMtxPiAndBinaryWeightsWithGraphicalStructure(
nStates,
thetaStationaryWeights,
thetaBinaryWeights,
bivariateFeatIndexDictionary=bivariateFeatDictionary)
initialStationaryDist = initialRateMtx.getStationaryDist()
initialExchangeCoef = initialRateMtx.getExchangeCoef()
theta0StationaryDistSamples[i, :] = initialStationaryDist
theta0ExchangeableCoefSamples[i, :] = initialExchangeCoef
weightGenerationRegime = WeightGenerationRegime(
nStates=nStates,
nBivariateFeat=nBivariateFeat,
stationaryWeights=thetaStationaryWeights,
bivariateWeights=thetaBinaryWeights)
prng = RandomState(np.random.choice(2**32 - 1, 1))
dataRegime = DataGenerationRegime(
nStates=nStates,
bivariateFeatIndexDictionary=bivariateFeatDictionary,
btLength=bt,
nSeq=nSeq,
weightGenerationRegime=weightGenerationRegime,
prng=prng,
interLength=interLength)
## generate the sequences data
seqList = dataRegime.generatingSeq()
suffStat = dataRegime.getSufficientStatFromSeq(seqList)
firstLastStatesArrayAll = dataRegime.generatingSeqGivenRateMtxAndBtInterval(
seqList)
# replicate K iterations to get new parameters
nTrans = suffStat["transitCount"]
holdTime = suffStat["sojourn"]
nInit = np.zeros(nStates)
unique, counts = np.unique(firstLastStatesArrayAll[0][:, 0],
return_counts=True)
nInitCount = np.asarray((unique, counts)).T
nInit[nInitCount[:, 0].astype(int)] = nInitCount[:, 1]
result = {}
result['stationaryDist'] = theta0StationaryDistSamples[(K - 1), :]
result['exchangeCoef'] = theta0ExchangeableCoefSamples[(K - 1), :]
result['stationaryWeights'] = thetaStationaryWeights
result['binaryWeights'] = thetaBinaryWeights
## after testing the code is right, the following two lines should be removed
#result['StationaryDistSamples'] = theta0StationaryDistSamples
#result['ExchangeableCoefSamples'] = theta0ExchangeableCoefSamples
return result
def run(self, uniWeightsValues=None, biWeightsValues=None):
## for every batchSize number of samples, we get the total sum
## of these samples and then refresh it to zero after we reach the
## batch size
## we only check the stationary distribution and exchangeable parameters
stationaryDistBatchSum = np.zeros((1, self.nStates))
exchangeCoefBatchSum = np.zeros((1, self.nExchange))
## output the true stationary distribution and exchangeable parameters
if self.unknownTrueRateMtx is False:
self.outputTrueParameters(self.dir_name)
# call methods to create initial samples
initialSamples = self.generateInitialSamples(
initialWeightsDist=self.initialWeightDist,
uniWeightsValues=uniWeightsValues,
biWeightsValues=biWeightsValues)
# decompose the initial samples
initialStationaryWeights = initialSamples['initialStationaryWeights']
initialStationaryDist = initialSamples['initialStationaryDist']
initialBinaryWeights = initialSamples['initialBinaryWeights']
initialRateMatrix = initialSamples['initialRateMatrix']
initialExchangeCoef = initialSamples['initialExchangeCoef']
# create arrays to save the posterior samples
posteriorSamples = self.generateArraysToSaveSamples()
stationaryDistSamples = posteriorSamples['stationaryDistSamples']
stationaryWeightsSamples = posteriorSamples['stationaryWeightsSamples']
binaryWeightsSamples = posteriorSamples['binaryWeightsSamples']
exchangeableSamples = posteriorSamples['exchangeableSamples']
rateMatrixSamples = posteriorSamples['rateMatrixSamples']
if self.dumpResultIteratively:
allFileNames = self.createAllOutputFileNames(
self.dir_name,
self.samplingMethod,
self.nMCMCIter,
saveRateMtx=False,
trajectoryLength=self.trajectoryLength,
mcmcSeed=self.initialSampleSeed)
# this algorithm runs a combination of HMC and local BPS
startTime = datetime.now()
if self.onlyHMC:
initializedPosteriorSamples = self.initializeHMCWeights(
initialStationaryWeights, initialBinaryWeights)
weightSamples = initializedPosteriorSamples['weightSamples']
sample = initializedPosteriorSamples['avgWeights']
else:
sample = initialStationaryWeights
previousStationaryDistMean = np.zeros((1, self.nStates))
previousExchangeCoefMean = np.zeros((1, self.nExchange))
nInit = np.zeros(self.nStates)
unique, counts = np.unique(self.data[0][:, 0], return_counts=True)
nInitCount = np.asarray((unique, counts)).T
nInit[nInitCount[:, 0].astype(int)] = nInitCount[:, 1]
for i in range(self.nMCMCIter):
stationaryWeightsSamples[i, :] = initialStationaryWeights
binaryWeightsSamples[i, :] = initialBinaryWeights
exchangeableSamples[i, :] = initialExchangeCoef
if self.saveRateMtx:
rateMatrixSamples[i, :] = initialRateMatrix
stationaryDistSamples[i, :] = initialStationaryDist
stationaryDistBatchSum = stationaryDistBatchSum + initialStationaryDist
exchangeCoefBatchSum = exchangeCoefBatchSum + initialExchangeCoef
if i > 0 and (i + 1) % self.batchSize == 0:
## When we reach the batch size, refresh the sum
## write the currrent file to csv and then refresh the vector to zeros
stationaryDistBatchMean = self.getErgodicMean(
nSamples=int(i + 1),
previousMean=previousStationaryDistMean,
newPartialSum=stationaryDistBatchSum,
batchSize=self.batchSize)
exchangeCoefBatchMean = self.getErgodicMean(
nSamples=int(i + 1),
previousMean=previousExchangeCoefMean,
newPartialSum=exchangeCoefBatchSum,
batchSize=self.batchSize)
previousStationaryDistMean = stationaryDistBatchMean
previousExchangeCoefMean = exchangeCoefBatchMean
self.dumpResult(stationaryDistBatchMean[0, :],
allFileNames['stationaryDistErgodicMean'])
self.dumpResult(exchangeCoefBatchMean[0, :],
allFileNames['exchangeCoefErgodicMean'])
stationaryDistBatchSum = np.zeros(self.nStates)
exchangeCoefBatchSum = np.zeros(self.nExchange)
if i > 0 and (i + 1) % self.dumpResultIterations == 0:
## record the results
self.dumpResult(
stationaryDistSamples[(i + 1 -
self.dumpResultIterations):(i +
1), :],
allFileNames['stationaryDist'])
self.dumpResult(
exchangeableSamples[(i + 1 -
self.dumpResultIterations):(i +
1), :],
allFileNames['exchangeableCoef'])
self.dumpResult(
binaryWeightsSamples[(i + 1 -
self.dumpResultIterations):(i +
1), :],
allFileNames['binaryWeights'])
self.dumpResult(
stationaryWeightsSamples[(
i + 1 - self.dumpResultIterations):(i + 1), :],
allFileNames['stationaryWeights'])
holdTime = np.zeros(self.nStates)
nTrans = np.zeros((self.nStates, self.nStates))
for j in range(self.dataGenerationRegime.nPairSeq):
## change it to the true rate matrix and see if the sufficient statistics match
#suffStat = FullTrajectorGeneration.endPointSamplerSummarizeStatisticsOneBt(True, self.prng,
# self.dataGenerationRegime.rateMtxObj.getRateMtx(),
# self.data[j],
# self.dataGenerationRegime.interLength)
suffStat = FullTrajectorGeneration.endPointSamplerSummarizeStatisticsOneBt(
True,
RandomState(int(i * self.dataGenerationRegime.nSeq + j)),
initialRateMatrix, self.data[j],
self.dataGenerationRegime.interLength)
# nInit = nInit + suffStat['nInit']
holdTime = holdTime + suffStat['holdTimes']
nTrans = nTrans + suffStat['nTrans']
# construct expected complete reversible model objective
if self.HMCPlusBPS:
expectedCompleteReversibleObjective = ExpectedCompleteReversibleObjective.ExpectedCompleteReversibleObjective(
holdTime, nInit, nTrans, 1.0, initialExchangeCoef)
if self.onlyHMC:
expectedCompleteReversibleObjective = ExpectedCompleteReversibleObjective.ExpectedCompleteReversibleObjective(
holdTime,
nInit,
nTrans,
1.0,
nBivariateFeatWeightsDictionary=self.
bivariateFeatIndexDictionary)
#####################################
hmc = HMC.HMC(self.nLeapFrogSteps, self.stepSize,
expectedCompleteReversibleObjective,
expectedCompleteReversibleObjective)
lastSample = sample
for k in range(self.nItersPerPathAuxVar):
hmcResult = hmc.doIter(self.nLeapFrogSteps, self.stepSize,
lastSample,
expectedCompleteReversibleObjective,
expectedCompleteReversibleObjective,
True)
lastSample = hmcResult.next_q
sample = lastSample
if self.onlyHMC:
initialStationaryWeights = sample[0:self.nStates]
initialBinaryWeights = sample[self.nStates:(
self.nStates + self.nBivariateFeat)]
# sample stationary distribution elements using HMC
if self.HMCPlusBPS:
initialStationaryWeights = sample
# update stationary distribution elements to the latest value
initialStationaryDist = np.exp(sample) / np.sum(np.exp(sample))
# sample exchangeable coefficients using local bouncy particle sampler
## define the model
model = ExpectedCompleteReversibleModelBinaryFactors.ExpectedCompleteReversibleModelWithBinaryFactors(
expectedCompleteReversibleObjective, self.nStates,
initialBinaryWeights, initialStationaryDist,
self.bivariateFeatIndexDictionary)
## define the sampler to use
## local sampler to use
if i == 0:
self.neighborVariablesForAllFactors = neighborVariableForAllFactors(
self.nStates, model.localFactors,
self.bivariateFeatIndexDictionary)
self.variableAndFactorInfo = neighbourVariblesAndFactorsAndExtendedNeighborsOfAllFactorsDict(
self.nStates, model.localFactors,
self.bivariateFeatIndexDictionary, self.nBivariateFeat)
self.indexOfFactorsForEachBivariateFeat = getIndexOfNeighborFactorsForEachIndexOfBinaryFeature(
self.bivariateFeatIndexDictionary, self.nBivariateFeat,
model.localFactors)
localSampler = LocalRFSamplerForBinaryWeights(
model, self.rfOptions, self.mcmcOptions, self.nStates,
self.neighborVariablesForAllFactors,
self.variableAndFactorInfo,
self.indexOfFactorsForEachBivariateFeat)
phyloLocalRFMove = PhyloLocalRFMove(
model=model,
sampler=localSampler,
initialPoints=initialBinaryWeights,
options=self.rfOptions,
prng=RandomState(i))
initialBinaryWeights = phyloLocalRFMove.execute()
initialRateMtx = ReversibleRateMtxPiAndBinaryWeightsWithGraphicalStructure(
self.nStates,
initialStationaryWeights,
initialBinaryWeights,
bivariateFeatIndexDictionary=self.bivariateFeatIndexDictionary)
initialStationaryDist = initialRateMtx.getStationaryDist()
initialRateMatrix = initialRateMtx.getRateMtx()
initialExchangeCoef = initialRateMtx.getExchangeCoef()
initialBinaryWeights = initialBinaryWeights
print("The current iteration finished is:")
print(i)
endTime = datetime.now()
timeElapsed = (endTime - startTime).total_seconds()
if not self.dumpResultIteratively:
result = {}
result['stationaryDistSamples'] = stationaryDistSamples
result['stationaryWeightsSamples'] = stationaryWeightsSamples
result['binaryWeightsSamples'] = binaryWeightsSamples
result['exchangeableSamples'] = exchangeableSamples
if self.saveRateMtx:
result['rateMatrixSamples'] = rateMatrixSamples
result['elapsingTime'] = timeElapsed
return result
else:
self.outputRunningTime(actualRunningTimeInSec=timeElapsed,
dir_name=self.dir_name,
time_base_filename="wallTime",
samplingMethod=self.samplingMethod,
nMCMCIter=self.nMCMCIter)