def analizeM():
M = MCMC(dm)
print("M: ", M)
M.sample(iter=10000, burn=1000, thin=10)
print("M t: ", M.trace('switchpoint')[:])
hist(M.trace('late_mean')[:])
# show()
plot(M)
# show()
print("M smd dm sp: ", M.step_method_dict[dm.switchpoint])
print("M smd dm em: ", M.step_method_dict[dm.early_mean])
print("M smd dm lm: ", M.step_method_dict[dm.late_mean])
M.use_step_method(Metropolis, dm.late_mean, proposal_sd=2.)
def runMCMCmodel(args):
"""
Simulate the survey data and run the MCMC luminosity calibration model.
Parameters
----------
args - Command line arguments
"""
mcmcParams = args['mcmcString']
surveyParams = args['surveyString']
priorParams = args['priorsString']
maxIter = int(mcmcParams[0])
burnIter = int(mcmcParams[1])
thinFactor = int(mcmcParams[2])
if surveyParams[5] == 'Inf':
magLim = np.Inf
else:
magLim = float(surveyParams[5])
S = U.UniformDistributionSingleLuminosity(int(surveyParams[0]),
float(surveyParams[1]),
float(surveyParams[2]),
float(surveyParams[3]),
float(surveyParams[4]),
surveyLimit=magLim)
#S.setRandomNumberSeed(53949896)
S.generateObservations()
lumCalModel = L.UniformSpaceDensityGaussianLF(S, float(surveyParams[1]),
float(surveyParams[2]),
float(priorParams[0]),
float(priorParams[1]),
float(priorParams[2]),
float(priorParams[3]))
class SurveyData(IsDescription):
"""
Class that holds the data model for the data from the simulated parallax survey. Intended for use
with the HDF5 files through the pytables package.
"""
trueParallaxes = Float64Col(S.numberOfStarsInSurvey)
absoluteMagnitudes = Float64Col(S.numberOfStarsInSurvey)
apparentMagnitudes = Float64Col(S.numberOfStarsInSurvey)
parallaxErrors = Float64Col(S.numberOfStarsInSurvey)
magnitudeErrors = Float64Col(S.numberOfStarsInSurvey)
observedParallaxes = Float64Col(S.numberOfStarsInSurvey)
observedMagnitudes = Float64Col(S.numberOfStarsInSurvey)
baseName = "LumCalSimSurvey-{0}".format(S.numberOfStars) + "-{0}".format(
S.minParallax)
baseName = baseName + "-{0}".format(S.maxParallax) + "-{0}".format(
S.meanAbsoluteMagnitude)
baseName = baseName + "-{0}".format(S.varianceAbsoluteMagnitude)
h5file = openFile(baseName + ".h5", mode="w", title="Simulated Survey")
group = h5file.createGroup("/", 'survey',
'Survey parameters, data, and MCMC parameters')
parameterTable = h5file.createTable(group, 'parameters', SurveyParameters,
"Survey parameters")
dataTable = h5file.createTable(group, 'data', SurveyData, "Survey data")
mcmcTable = h5file.createTable(group, 'mcmc', McmcParameters,
"MCMC parameters")
surveyParams = parameterTable.row
surveyParams['kind'] = S.__class__.__name__
surveyParams['numberOfStars'] = S.numberOfStars
surveyParams['minParallax'] = S.minParallax
surveyParams['maxParallax'] = S.maxParallax
surveyParams['meanAbsoluteMagnitude'] = S.meanAbsoluteMagnitude
surveyParams['varianceAbsoluteMagnitude'] = S.varianceAbsoluteMagnitude
surveyParams[
'parallaxErrorNormalizationMagnitude'] = S.parallaxErrorNormalizationMagnitude
surveyParams['parallaxErrorSlope'] = S.parallaxErrorSlope
surveyParams[
'parallaxErrorCalibrationFloor'] = S.parallaxErrorCalibrationFloor
surveyParams[
'magnitudeErrorNormalizationMagnitude'] = S.magnitudeErrorNormalizationMagnitude
surveyParams['magnitudeErrorSlope'] = S.magnitudeErrorSlope
surveyParams[
'magnitudeErrorCalibrationFloor'] = S.magnitudeErrorCalibrationFloor
surveyParams['apparentMagnitudeLimit'] = S.apparentMagnitudeLimit
surveyParams['numberOfStarsInSurvey'] = S.numberOfStarsInSurvey
surveyParams.append()
parameterTable.flush()
surveyData = dataTable.row
surveyData['trueParallaxes'] = S.trueParallaxes
surveyData['absoluteMagnitudes'] = S.absoluteMagnitudes
surveyData['apparentMagnitudes'] = S.apparentMagnitudes
surveyData['parallaxErrors'] = S.parallaxErrors
surveyData['magnitudeErrors'] = S.magnitudeErrors
surveyData['observedParallaxes'] = S.observedParallaxes
surveyData['observedMagnitudes'] = S.observedMagnitudes
surveyData.append()
dataTable.flush()
mcmcParameters = mcmcTable.row
mcmcParameters['iterations'] = maxIter
mcmcParameters['burnIn'] = burnIter
mcmcParameters['thin'] = thinFactor
mcmcParameters['minMeanAbsoluteMagnitude'] = float(priorParams[0])
mcmcParameters['maxMeanAbsoluteMagnitude'] = float(priorParams[1])
mcmcParameters['priorTau'] = "Inverse-Gamma"
mcmcParameters['shapeTau'] = float(priorParams[2])
mcmcParameters['scaleTau'] = float(priorParams[3])
mcmcParameters.append()
dataTable.flush()
h5file.close()
# Run MCMC and store in HDF5 database
baseName = "LumCalResults-{0}".format(S.numberOfStars) + "-{0}".format(
S.minParallax)
baseName = baseName + "-{0}".format(S.maxParallax) + "-{0}".format(
S.meanAbsoluteMagnitude)
baseName = baseName + "-{0}".format(S.varianceAbsoluteMagnitude)
M = MCMC(lumCalModel.pyMCModel,
db='hdf5',
dbname=baseName + ".h5",
dbmode='w',
dbcomplevel=9,
dbcomplib='bzip2')
M.use_step_method(Metropolis, M.priorParallaxes)
M.use_step_method(Metropolis, M.priorAbsoluteMagnitudes)
start = now()
M.sample(iter=maxIter, burn=burnIter, thin=thinFactor)
finish = now()
print "Elapsed time in seconds: %f" % (finish - start)
M.db.close()
def runMCMCmodel(args):
"""
Simulate the survey data and run the MCMC luminosity calibration model.
Parameters
----------
args - Command line arguments
"""
mcmcParams=args['mcmcString']
surveyParams=args['surveyString']
priorParams=args['priorsString']
maxIter=int(mcmcParams[0])
burnIter=int(mcmcParams[1])
thinFactor=int(mcmcParams[2])
if surveyParams[5] == 'Inf':
magLim = np.Inf
else:
magLim = float(surveyParams[5])
S=U.UniformDistributionSingleLuminosity(int(surveyParams[0]), float(surveyParams[1]),
float(surveyParams[2]), float(surveyParams[3]), float(surveyParams[4]),
surveyLimit=magLim)
#S.setRandomNumberSeed(53949896)
S.generateObservations()
lumCalModel=L.UniformSpaceDensityGaussianLFBook(S,float(surveyParams[1]), float(surveyParams[2]),
float(priorParams[0]), float(priorParams[1]), float(priorParams[2]), float(priorParams[3]))
class SurveyData(IsDescription):
"""
Class that holds the data model for the data from the simulated parallax survey. Intended for use
with the HDF5 files through the pytables package.
"""
trueParallaxes = Float64Col(S.numberOfStarsInSurvey)
absoluteMagnitudes = Float64Col(S.numberOfStarsInSurvey)
apparentMagnitudes = Float64Col(S.numberOfStarsInSurvey)
parallaxErrors = Float64Col(S.numberOfStarsInSurvey)
magnitudeErrors = Float64Col(S.numberOfStarsInSurvey)
observedParallaxes = Float64Col(S.numberOfStarsInSurvey)
observedMagnitudes = Float64Col(S.numberOfStarsInSurvey)
baseName="LumCalSimSurvey-{0}".format(S.numberOfStars)+"-{0}".format(S.minParallax)
baseName=baseName+"-{0}".format(S.maxParallax)+"-{0}".format(S.meanAbsoluteMagnitude)
baseName=baseName+"-{0}".format(S.varianceAbsoluteMagnitude)
h5file = openFile(baseName+".h5", mode = "w", title = "Simulated Survey")
group = h5file.createGroup("/", 'survey', 'Survey parameters, data, and MCMC parameters')
parameterTable = h5file.createTable(group, 'parameters', SurveyParameters, "Survey parameters")
dataTable = h5file.createTable(group, 'data', SurveyData, "Survey data")
mcmcTable = h5file.createTable(group, 'mcmc', McmcParameters, "MCMC parameters")
surveyParams = parameterTable.row
surveyParams['kind']=S.__class__.__name__
surveyParams['numberOfStars']=S.numberOfStars
surveyParams['minParallax']=S.minParallax
surveyParams['maxParallax']=S.maxParallax
surveyParams['meanAbsoluteMagnitude']=S.meanAbsoluteMagnitude
surveyParams['varianceAbsoluteMagnitude']=S.varianceAbsoluteMagnitude
surveyParams['parallaxErrorNormalizationMagnitude']=S.parallaxErrorNormalizationMagnitude
surveyParams['parallaxErrorSlope']=S.parallaxErrorSlope
surveyParams['parallaxErrorCalibrationFloor']=S.parallaxErrorCalibrationFloor
surveyParams['magnitudeErrorNormalizationMagnitude']=S.magnitudeErrorNormalizationMagnitude
surveyParams['magnitudeErrorSlope']=S.magnitudeErrorSlope
surveyParams['magnitudeErrorCalibrationFloor']=S.magnitudeErrorCalibrationFloor
surveyParams['apparentMagnitudeLimit']=S.apparentMagnitudeLimit
surveyParams['numberOfStarsInSurvey']=S.numberOfStarsInSurvey
surveyParams.append()
parameterTable.flush()
surveyData = dataTable.row
surveyData['trueParallaxes']=S.trueParallaxes
surveyData['absoluteMagnitudes']=S.absoluteMagnitudes
surveyData['apparentMagnitudes']=S.apparentMagnitudes
surveyData['parallaxErrors']=S.parallaxErrors
surveyData['magnitudeErrors']=S.magnitudeErrors
surveyData['observedParallaxes']=S.observedParallaxes
surveyData['observedMagnitudes']=S.observedMagnitudes
surveyData.append()
dataTable.flush()
mcmcParameters = mcmcTable.row
mcmcParameters['iterations']=maxIter
mcmcParameters['burnIn']=burnIter
mcmcParameters['thin']=thinFactor
mcmcParameters['minMeanAbsoluteMagnitude']=float(priorParams[0])
mcmcParameters['maxMeanAbsoluteMagnitude']=float(priorParams[1])
mcmcParameters['priorTau']="OneOverX"
mcmcParameters['tauLow']=float(priorParams[2])
mcmcParameters['tauHigh']=float(priorParams[3])
mcmcParameters.append()
dataTable.flush()
h5file.close()
# Run MCMC and store in HDF5 database
baseName="LumCalResults-{0}".format(S.numberOfStars)+"-{0}".format(S.minParallax)
baseName=baseName+"-{0}".format(S.maxParallax)+"-{0}".format(S.meanAbsoluteMagnitude)
baseName=baseName+"-{0}".format(S.varianceAbsoluteMagnitude)
M=MCMC(lumCalModel.pyMCModel, db='hdf5', dbname=baseName+".h5", dbmode='w', dbcomplevel=9,
dbcomplib='bzip2')
M.use_step_method(Metropolis, M.priorParallaxes)
M.use_step_method(Metropolis, M.priorAbsoluteMagnitudes)
start=now()
M.sample(iter=maxIter, burn=burnIter, thin=thinFactor)
finish=now()
print "Elapsed time in seconds: %f" % (finish-start)
M.db.close()
# M.sample(iter=400000, burn=50000, thin=10,verbose=0)
# np.save('mc_data/nm_rho_s.npy',M.trace('rho_s')[:])
# np.save('mc_data/nm_alpha.npy',M.trace('alpha')[:])
# np.save('mc_data/nm_beta.npy',M.trace('beta')[:])
# np.save('mc_data/nm_ia.npy',M.trace('interaction_angle')[:])
# np.save('mc_data/nm_il.npy',M.trace('interaction_length')[:])
# np.save('mc_data/nm_ig.npy',M.trace('ignore_length')[:])
# network model with alignment
M = MCMC(networkModelAlignMC)
M.use_step_method(
pymc.AdaptiveMetropolis,
[
networkModelAlignMC.interaction_angle,
networkModelAlignMC.interaction_length,
networkModelAlignMC.align_weight,
networkModelAlignMC.ignore_length,
],
delay=1000,
)
M.sample(iter=400000, burn=50000, thin=10, verbose=0)
np.save("mc_data/nma_rho_s.npy", M.trace("rho_s")[:])
np.save("mc_data/nma_alpha.npy", M.trace("alpha")[:])
np.save("mc_data/nma_beta.npy", M.trace("beta")[:])
np.save("mc_data/nma_ia.npy", M.trace("interaction_angle")[:])
np.save("mc_data/nma_il.npy", M.trace("interaction_length")[:])
np.save("mc_data/nma_aw.npy", M.trace("align_weight")[:])
np.save("mc_data/nma_ig.npy", M.trace("ignore_length")[:])
@deterministic(plot=False)
def r(s=s, e=e, l=l):
"""Allocate appropriate mean to time series"""
out = np.empty(len(disasters_array))
# Early mean prior to switchpoint
out[:s] = e
# Late mean following switchpoint
out[s:] = l
return out
# Where the value of x is None, the value is taken as missing.
D = Impute('D', Poisson, x, mu=r)
M.step_method_dict[DisasterModel.s]
#[<pymc.StepMethods.DiscreteMetropolis object at 0x3e8cb50>]
M.step_method_dict[DisasterModel.e]
#[<pymc.StepMethods.Metropolis object at 0x3e8cbb0>]
M.step_method_dict[DisasterModel.l]
#[<pymc.StepMethods.Metropolis object at 0x3e8ccb0>]
from pymc import Metropolis
M.use_step_method(Metropolis, DisasterModel.l, proposal_sd=2.)
#
# if np.log(random.random()) > logp_p - logp:
#
# self.reject()
return locals()
if __name__ == '__main__':
model = make_bma()
M = MCMC(model)
M.use_step_method(model['ModelMetropolis'], model['regression_model'])
M.sample(iter=5000, burn=1000, thin=1)
model_chain = M.trace("regression_model")[:]
from collections import Counter
counts = Counter(model_chain).items()
counts.sort(reverse=True, key=lambda x: x[1])
for f in counts[:10]:
columns = unpack(f[0])
print('Visits:', f[1])
print(np.array([1. if i in columns else 0 for i in range(0,M.rank)]))
print(M.coefficients.flatten())
X = sm.add_constant(M.X[:, columns], prepend=True)
def run_model(mtype,
msubtype,
model_params,
desc,
niter=20000,
nburnin=15000,
nthin=5,
nchains=3,
am_delay=5000,
am_interval=1000,
burn_till_tuned=True):
"""
Run PYMC model
"""
model = spat_nmixture_nb
curr_dir = os.getcwd()
output_dir = os.path.join(curr_dir, 'run', mtype, msubtype, desc)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
##### ---- RUN MODEL ---- #####
os.chdir(output_dir)
trace_file = os.path.join(output_dir, desc + '.hdf5')
if os.path.exists(trace_file):
os.remove(trace_file)
while 1:
try:
M = MCMC(model.build_model(**model_params),
db='hdf5',
dbname=trace_file)
break
except ZeroProbability:
model_params.update({
'ls_beta_inits':
np.random.normal(0, 1, model_params['ls_dmat'].shape[1]),
'beta_inits':
np.random.normal(0, 1, model_params['dmat'].shape[1])
})
M.use_step_method(AdaptiveMetropolis,
M.ls_beta,
delay=am_delay,
interval=am_interval)
M.use_step_method(AdaptiveMetropolis,
M.beta,
delay=am_delay,
interval=am_interval)
M.use_step_method(AdaptiveMetropolis,
M.eps,
delay=am_delay,
interval=am_interval)
M.use_step_method(AdaptiveMetropolis,
M.alpha,
delay=am_delay,
interval=am_interval)
M.sample(niter, burn=nburnin, thin=nthin, burn_till_tuned=burn_till_tuned)
for i in range(nchains - 1):
M.sample(niter, burn=nburnin, thin=nthin)
M.db.close()
os.chdir(curr_dir)
return 0
e = Exponential('e', beta=1)
# Late mean
l = Exponential('l', beta=1)
@deterministic(plot=False)
def r(s=s, e=e, l=l):
"""Allocate appropriate mean to time series"""
out = np.empty(len(disasters_array))
# Early mean prior to switchpoint
out[:s] = e
# Late mean following switchpoint
out[s:] = l
return out
# Where the value of x is None, the value is taken as missing.
D = Impute('D', Poisson, x, mu=r)
M.step_method_dict[DisasterModel.s]
#[<pymc.StepMethods.DiscreteMetropolis object at 0x3e8cb50>]
M.step_method_dict[DisasterModel.e]
#[<pymc.StepMethods.Metropolis object at 0x3e8cbb0>]
M.step_method_dict[DisasterModel.l]
#[<pymc.StepMethods.Metropolis object at 0x3e8ccb0>]
from pymc import Metropolis
M.use_step_method(Metropolis, DisasterModel.l, proposal_sd=2.)
E = l * u / (n * Z) * kxf(px, kxmax, p50) * (psi - px) / 1000
s[i] = min(sp - E + Rfi / 1000 / n / Z, 1)
sapflow_modeled.append(E / alpha)
else:
print('gs = 0')
s[i] = min(sp + Rfi / 1000 / n / Z, 1)
sapflow_modeled.append(0)
return sapflow_modeled
'''data likelihoods'''
np.random.seed(1)
Y_obs = Normal('Y_obs', mu=muf, tau=sigma, value=vn, observed=True)
''' posterior sampling '''
M = MCMC([alpha, c, g1, kxmax, Lamp, Lave, LTf, p50, Z, sigma])
M.use_step_method(AdaptiveMetropolis,
[alpha, c, g1, kxmax, Lamp, Lave, LTf, p50, Z, sigma])
M.sample(iter=1000000, burn=500000, thin=40)
# Save trace
ensure_dir(species)
traces = {
'alpha': M.trace('alpha')[:],
'c': M.trace('c')[:],
'g1': M.trace('g1')[:],
'kxmax': M.trace('kxmax')[:],
'Lamp': M.trace('Lamp')[:],
'Lave': M.trace('Lave')[:],
'LTf': M.trace('LTf')[:],
'p50': M.trace('p50')[:],
'Z': M.trace('Z')[:],
'sigma': M.trace('sigma')[:]
