def test_fit(self):
p = self._build_parent()
s = MyStochastic(self.STOCHASTIC_NAME, p)
mcmc = MCMC({p, s})
mcmc.sample(100, burn=10, thin=2)
def test_simple(self):
# Priors
mu = Normal('mu', mu=0, tau=0.0001)
s = Uniform('s', lower=0, upper=100, value=10)
tau = s ** -2
# Likelihood with missing data
x = Normal('x', mu=mu, tau=tau, value=m, observed=True)
# Instantiate sampler
M = MCMC([mu, s, tau, x])
# Run sampler
M.sample(10000, 5000, progress_bar=0)
# Check length of value
assert_equal(len(x.value), 100)
# Check size of trace
tr = M.trace('x')()
assert_equal(shape(tr), (5000, 2))
sd2 = [-2 < i < 2 for i in ravel(tr)]
# Check for standard normal output
assert_almost_equal(sum(sd2) / 10000., 0.95, decimal=1)
def test_nd(self):
M = MCMC([self.NDstoch()], db=self.name, dbname=os.path.join(testdir, 'ND.'+self.name), dbmode='w')
M.sample(10, progress_bar=0)
a = M.trace('nd')[:]
assert_equal(a.shape, (10,2,2))
db = getattr(pymc.database, self.name).load(os.path.join(testdir, 'ND.'+self.name))
assert_equal(db.trace('nd')[:], a)
def estimate_failures(samples, #samples from noisy labelers
n_samples=10000, #number of samples to run MCMC for
burn=None, #burn-in. Defaults to n_samples/2
thin=10, #thinning rate. Sample every k samples from markov chain
alpha_p=1, beta_p=1, #beta parameters for true positive rate
alpha_e=1, beta_e=10 #beta parameters for noise rates
):
if burn is None:
burn = n_samples / 2
S,N = samples.shape
p = Beta('p', alpha=alpha_p, beta=beta_p) #prior on true label
l = Bernoulli('l', p=p, size=S)
e_pos = Beta('e_pos', alpha_e, beta_e, size=N) # error rate if label = 1
e_neg = Beta('e_neg', alpha_e, beta_e, size=N) # error rate if label = 0
@deterministic(plot=False)
def noise_rate(l=l, e_pos=e_pos, e_neg=e_neg):
#probability that a noisy labeler puts a label 1
return np.outer(l, 1-e_pos) + np.outer(1-l, e_neg)
noisy_label = Bernoulli('noisy_label', p=noise_rate, size=samples.shape, value=samples, observed=True)
variables = [l, e_pos, e_neg, p, noisy_label, noise_rate]
model = MCMC(variables, verbose=3)
model.sample(iter=n_samples, burn=burn, thin=thin)
model.write_csv('out.csv', ['p', 'e_pos', 'e_neg'])
p = np.median(model.trace('p')[:])
e_pos = np.median(model.trace('e_pos')[:],0)
e_neg = np.median(model.trace('e_neg')[:],0)
return p, e_pos, e_neg
def test_pymc_model(self):
""" Tests sampler """
sampler = MCMC(model_omm.pymc_parameters)
self.assert_(isinstance(model_omm, TorsionFitModelOMM))
self.assert_(isinstance(sampler, pymc.MCMC))
sampler.sample(iter=1)
def test_fit_with_sibling(self):
p = self._build_parent()
s = MyStochastic(self.STOCHASTIC_NAME, p)
sib = MyStochastic(self.SIBLING_NAME, p)
mcmc = MCMC({p, s, sib})
mcmc.sample(100, burn=10, thin=2)
def mcmc(prob, nsample=100, modulename = 'model' ):
try:
mystr = "from " + modulename + " import model"
exec(mystr)
except:
print 'cannot import', modulename
M = MCMC( model(prob) )
M.sample(nsample)
return M
def test_zcompression(self):
db = pymc.database.hdf5.Database(dbname=os.path.join(testdir, 'DisasterModelCompressed.hdf5'),
dbmode='w',
dbcomplevel=5)
S = MCMC(DisasterModel, db=db)
S.sample(45,10,1)
assert_array_equal(S.e.trace().shape, (35,))
S.db.close()
db.close()
del S
def test_zcompression(self):
with warnings.catch_warnings():
warnings.simplefilter('ignore')
db = pymc.database.hdf5.Database(dbname=os.path.join(testdir, 'disaster_modelCompressed.hdf5'),
dbmode='w',
dbcomplevel=5)
S = MCMC(disaster_model, db=db)
S.sample(45,10,1, progress_bar=0)
assert_array_equal(S.trace('early_mean')[:].shape, (35,))
S.db.close()
db.close()
del S
def MCMC( self, nruns=10000, burn=1000, init_error_std=1., max_error_std=100., verbose=1 ):
''' Perform Markov Chain Monte Carlo sampling using pymc package
:param nruns: Number of MCMC iterations (samples)
:type nruns: int
:param burn: Number of initial samples to burn (discard)
:type burn: int
:param verbose: verbosity of output
:type verbose: int
:param init_error_std: Initial standard deviation of residuals
:type init_error_std: fl64
:param max_error_std: Maximum standard deviation of residuals that will be considered
:type max_error_std: fl64
:returns: pymc MCMC object
'''
if max_error_std < init_error_std:
print "Error: max_error_std must be greater than or equal to init_error_std"
return
try:
from pymc import Uniform, deterministic, Normal, MCMC, Matplot
except ImportError as exc:
sys.stderr.write("Warning: failed to import pymc module. ({})\n".format(exc))
sys.stderr.write("If pymc is not installed, try installing:\n")
sys.stderr.write("e.g. try using easy_install: easy_install pymc\n")
def __mcmc_model( self, init_error_std=1., max_error_std=100. ):
#priors
variables = []
sig = Uniform('error_std', 0.0, max_error_std, value=init_error_std)
variables.append( sig )
for nm,mn,mx in zip(self.parnames,self.parmins,self.parmaxs):
evalstr = "Uniform( '" + str(nm) + "', " + str(mn) + ", " + str(mx) + ")"
variables.append( eval(evalstr) )
#model
@deterministic()
def residuals( pars = variables, p=self ):
values = []
for i in range(1,len(pars)):
values.append(float(pars[i]))
pardict = dict(zip(p.parnames,values))
p.forward(pardict=pardict, reuse_dirs=True)
return numpy.array(p.residuals)*numpy.array(p.obsweights)
#likelihood
y = Normal('y', mu=residuals, tau=1.0/sig**2, observed=True, value=numpy.zeros(len(self.obs)))
variables.append(y)
return variables
M = MCMC( __mcmc_model(self, init_error_std=init_error_std, max_error_std=max_error_std) )
M.sample(iter=nruns,burn=burn,verbose=verbose)
return M
def analizeMwm():
masked_values = np.ma.masked_equal(x, value=None)
print("m v: ", masked_values)
print("dmwm da: ", dmwm.disasters_array)
Mwm = MCMC(dmwm)
Mwm.sample(iter=10000, burn=1000, thin=10)
print("Mwm t: ", Mwm.trace('switchpoint')[:])
hist(Mwm.trace('late_mean')[:])
# show()
plot(Mwm)
def test_zcompression(self):
original_filters = warnings.filters[:]
warnings.simplefilter("ignore")
try:
db = pymc.database.hdf5.Database(dbname=os.path.join(testdir, 'disaster_modelCompressed.hdf5'),
dbmode='w',
dbcomplevel=5)
S = MCMC(disaster_model, db=db)
S.sample(45,10,1, progress_bar=0)
assert_array_equal(S.trace('early_mean')[:].shape, (35,))
S.db.close()
db.close()
del S
finally:
warnings.filters = original_filters
def bimodal_gauss(data,pm,dmin=0.3):
'''run MCMC to get regression on bimodal normal distribution'''
size = len(data[pm])
### set up model
p = Uniform( "p", 0.2 , 0.8) #this is the fraction that come from mean1 vs mean2
# p = distributions.truncated_normal_like('p', mu=0.5, tau=0.001, a=0., b=1.)
# p = Normal( 'p', mu=(1.*sum(comp0==1))/size, tau=1./0.1**2 ) # attention: wings!, tau = 1/sig^2
# p = Normal( 'p', mu=0.5, tau=1./0.1**2 ) # attention: wings!, tau = 1/sig^2
ber = Bernoulli( "ber", p = p, size = size) # produces 1 with proportion p
precision = Gamma('precision', alpha=0.01, beta=0.01)
mean1 = Uniform( "mean1", -0.5, 1.0) # if not truncated
sig1 = Uniform( 'sig1', 0.01, 1.)
mean2 = Uniform( "mean2", mean1 + dmin, 1.5)
sig2 = Uniform( 'sig2', 0.01, 1.)
pop1 = Normal( 'pop1', mean1, 1./sig1**2) # tau is 1/sig^2
pop2 = Normal( 'pop2', mean2, 1./sig2**2)
@deterministic
def bimod(ber = ber, pop1 = pop1, pop2 = pop2): # value determined from parents completely
return ber*pop1 + (1-ber)*pop2
obs = Normal( "obs", bimod, precision, value = data[pm], observed = True)
model = Model( {"p":p, "precision": precision, "mean1": mean1, 'sig1': sig1, "mean2":mean2, 'sig2':sig2, "obs":obs} )
from pymc import MCMC, Matplot
M = MCMC(locals(), db='pickle', dbname='metals.pickle')
iter = 10000; burn = 9000; thin = 10
M.sample(iter=iter, burn=burn, thin=thin)
M.db.commit()
mu1 = np.mean(M.trace('mean1')[:])
sig1= np.mean(M.trace('sig1')[:])
mu2 = np.mean(M.trace('mean2')[:])
sig2= np.mean(M.trace('sig2')[:])
p = np.mean(M.trace('p')[:])
return p, mu1, sig1, mu2, sig2, M
def analizeM():
M = MCMC(dm)
print("M: ", M)
M.sample(iter=10000, burn=1000, thin=10)
print("M t: ", M.trace('switchpoint')[:])
hist(M.trace('late_mean')[:])
# show()
plot(M)
# show()
print("M smd dm sp: ", M.step_method_dict[dm.switchpoint])
print("M smd dm em: ", M.step_method_dict[dm.early_mean])
print("M smd dm lm: ", M.step_method_dict[dm.late_mean])
M.use_step_method(Metropolis, dm.late_mean, proposal_sd=2.)
def bimodal_gauss(data,pm):
'''run MCMC to get regression on bimodal normal distribution'''
m1 = np.mean(data[pm])/2.
m2 = np.mean(data[pm])*2.
dm = m2 - m1
size = len(data[pm])
### set up model
p = Uniform( "p", 0.2 , 0.8) #this is the fraction that come from mean1 vs mean2
# p = distributions.truncated_normal_like('p', mu=0.5, tau=0.001, a=0., b=1.)
# p = Normal( 'p', mu=(1.*sum(comp0==1))/size, tau=1./0.1**2 ) # attention: wings!, tau = 1/sig^2
# p = Normal( 'p', mu=0.5, tau=1./0.1**2 ) # attention: wings!, tau = 1/sig^2
ber = Bernoulli( "ber", p = p, size = size) # produces 1 with proportion p
precision = Gamma('precision', alpha=0.01, beta=0.01)
dmu = Normal( 'dmu', dm, tau=1./0.05**2 ) # [PS] give difference between means, finite
# dmu = Lognormal( 'dmu', 0.3, tau=1./0.1**2)
mean1 = Normal( "mean1", mu = m1, tau = 1./0.1**2 ) # better to use Normals versus Uniforms,
# if not truncated
mean2 = Normal( "mean2", mu = mean1 + dmu, tau = 1./0.1**2 ) # tau is 1/sig^2
@deterministic
def mean( ber = ber, mean1 = mean1, mean2 = mean2):
return ber*mean1 + (1-ber)*mean2
obs = Normal( "obs", mean, precision, value = data[pm], observed = True)
model = Model( {"p":p, "precision": precision, "mean1": mean1, "mean2":mean2, "obs":obs} )
from pymc import MCMC, Matplot
M = MCMC(locals(), db='pickle', dbname='metals.pickle')
iter = 3000; burn = 2000; thin = 10
M.sample(iter=iter, burn=burn, thin=thin)
M.db.commit()
mu1 = np.mean(M.trace('mean1')[:])
mu2 = np.mean(M.trace('mean2')[:])
p = np.mean(M.trace('p')[:])
return p, mu1, 0.1, mu2, 0.1, M
class LeagueModel(object):
"""MCMC model of a football league."""
def __init__(self, fname):
super(LeagueModel, self).__init__()
league = fuba.League(fname)
N = len(league.teams)
#dummy future games
future_games = [[league.teams["Werder Bremen"],league.teams["Dortmund"]]]
self.goal_rate = np.empty(N,dtype=object)
self.match_rate = np.empty(len(league.games)*2,dtype=object)
self.match_goals_future = np.empty(len(future_games)*2,dtype=object)
self.home_adv = Normal(name = 'home_adv',mu=0,tau=10.)
for t in league.teams.values():
print t.name,t.team_id
self.goal_rate[t.team_id] = Exponential('goal_rate_%i'%t.team_id,beta=1)
for game in range(len(league.games)):
self.match_rate[2*game] = Poisson('match_rate_%i'%(2*game),
mu=self.goal_rate[league.games[game].hometeam.team_id] + self.home_adv,
value=league.games[game].homescore, observed=True)
self.match_rate[2*game+1] = Poisson('match_rate_%i'%(2*game+1),
mu=self.goal_rate[league.games[game].hometeam.team_id],
value=league.games[game].homescore, observed=True)
for game in range(len(future_games)):
self.match_goals_future[2*game] = Poisson('match_goals_future_%i'%(2*game),
mu=self.goal_rate[future_games[game][0].team_id] + self.home_adv)
self.match_goals_future[2*game+1] = Poisson('match_goals_future_%i'%(2*game+1),
mu=self.goal_rate[future_games[game][1].team_id])
def run_mc(self,nsample = 10000,interactive=False):
"""run the model using mcmc"""
from pymc.Matplot import plot
from pymc import MCMC
self.M = MCMC(self)
if interactive:
self.M.isample(iter=nsample, burn=1000, thin=10)
else:
self.M.sample(iter=nsample, burn=1000, thin=10)
plot(self.M)
def bayesian_regression(self, Methodology):
fit_dict = OrderedDict()
fit_dict['methodology'] = r'Inference $\chi^{2}$ model'
#Initial guess for the fitting:
Np_lsf = polyfit(self.x_array, self.y_array, 1)
m_0, n_0 = Np_lsf[0], Np_lsf[1]
MCMC_dict = self.lr_ChiSq(self.x_array, self.y_array, m_0, n_0)
myMCMC = MCMC(MCMC_dict)
myMCMC.sample(iter=10000, burn=1000)
fit_dict['m'], fit_dict['n'], fit_dict['m_error'], fit_dict['n_error'] = myMCMC.stats()['m']['mean'], myMCMC.stats()['n']['mean'], myMCMC.stats()['m']['standard deviation'], myMCMC.stats()['n']['standard deviation']
return fit_dict
def fit_std_curve_by_pymc(i_vals, i_sds, dpx_concs):
import pymc
from pymc import Uniform, stochastic, deterministic, MCMC
from pymc import Matplot
# Define prior distributions for both Ka and Kd
ka = Uniform('ka', lower=0, upper=1000)
kd = Uniform('kd', lower=0, upper=1000)
@stochastic(plot=True, observed=True)
def quenching_model(ka=ka, kd=kd, value=i_vals):
pred_i = quenching_func(ka, kd, dpx_concs)
# The first concentration in dpx_concs should always be zero
# (that is, the first point in the titration should be the
# unquenched fluorescence), so we assert that here:
assert dpx_concs[0] == 0
# The reason this is necessary is that in the likelihood calculation
# we skip the error for the first point, since (when the std. err
# is calculated by well) the error is 0 (the I / I_0 ratio is
# always 1 for each well, the the variance/SD across the wells is 0).
# If we don't skip this first point, we get nan for the likelihood.
# In addition, the model always predicts 1 for the I / I_0 ratio
# when the DPX concentration is 0, so it contributes nothing to
# the overall fit.
return -np.sum((value[1:] - pred_i[1:])**2 / (2 * i_sds[1:]**2))
pymc_model = pymc.Model([ka, kd, quenching_model])
mcmc = MCMC(pymc_model)
mcmc.sample(iter=155000, burn=5000, thin=150)
Matplot.plot(mcmc)
plt.figure()
num_to_plot = 1000
ka_vals = mcmc.trace('ka')[:]
kd_vals = mcmc.trace('kd')[:]
if num_to_plot > len(ka_vals):
num_to_plot = len(ka_vals)
for i in range(num_to_plot):
plt.plot(dpx_concs, quenching_func(ka_vals[i], kd_vals[i], dpx_concs),
alpha=0.01, color='r')
plt.errorbar(dpx_concs, i_vals, yerr=i_sds, linestyle='', marker='o',
color='k', linewidth=2)
return (ka_vals, kd_vals)
def estimate_failures_from_counts(counts, #samples from noisy labelers
n_samples=10000, #number of samples to run MCMC for
burn=None, #burn-in. Defaults to n_samples/2
thin=10, #thinning rate. Sample every k samples from markov chain
alpha_p=1, beta_p=1, #beta parameters for true positive rate
alpha_e=1, beta_e=10 #beta parameters for noise rates
):
if burn is None:
burn = n_samples / 2
S = counts.sum()
N = len(counts.shape)
p_label = Beta('p_label', alpha=alpha_p, beta=beta_p) #prior on true label
e_pos = Beta('e_pos', alpha_e, beta_e, size=N) # error rate if label = 1
e_neg = Beta('e_neg', alpha_e, beta_e, size=N) # error rate if label = 0
print counts
@deterministic(plot=False)
def patterns(p_label=p_label, e_pos=e_pos, e_neg=e_neg):
#probability that the noisy labelers output pattern p
P = np.zeros((2,)*N)
for pat in itertools.product([0,1], repeat=N):
P[pat] = p_label*np.product([1-e_pos[i] if pat[i]==1 else e_pos[i] for i in xrange(N)])
P[pat] += (1-p_label)*np.product([e_neg[i] if pat[i]==1 else 1-e_neg[i] for i in xrange(N)])
assert np.abs(P.sum() - 1) < 1e-6
return P.ravel()
pattern_counts = Multinomial('pattern_counts',n=S, p=patterns, value=counts.ravel(), observed=True)
variables = [p_label, e_pos, e_neg, patterns]
model = MCMC(variables, verbose=3)
model.sample(iter=n_samples, burn=burn, thin=thin)
model.write_csv('out.csv', ['p_label', 'e_pos', 'e_neg'])
p = np.median(model.trace('p_label')[:])
e_pos = np.median(model.trace('e_pos')[:],0)
e_neg = np.median(model.trace('e_neg')[:],0)
return p, e_pos, e_neg
def test_non_missing(self):
"""
Test to ensure that masks without any missing values are not imputed.
"""
fake_data = rnormal(0, 1, size=10)
m = ma.masked_array(fake_data, fake_data == -999)
# Priors
mu = Normal('mu', mu=0, tau=0.0001)
s = Uniform('s', lower=0, upper=100, value=10)
tau = s ** -2
# Likelihood with missing data
x = Normal('x', mu=mu, tau=tau, value=m, observed=True)
# Instantiate sampler
M = MCMC([mu, s, tau, x])
# Run sampler
M.sample(20000, 19000, progress_bar=0)
# Ensure likelihood does not have a trace
assert_raises(AttributeError, x.__getattribute__, 'trace')
def Outliers_Krough(self):
fit_dict = OrderedDict()
fit_dict['methodology'] = r'Outliers Krough'
#Initial Guess for fitting
Bces_guess = self.bces_regression()
m_0, n_0 = Bces_guess['m'][0], Bces_guess['n'][0]
Spread_vector = ones(len(self.x_array))
#Model for outliers detection
Outliers_dect_dict = self.inference_outliers(self.x_array, self.y_array, m_0, n_0, Spread_vector)
mcmc = MCMC(Outliers_dect_dict)
mcmc.sample(100000, 20000)
#Extract the data with the outliers coordinates
probability_of_points = mcmc.trace('inlier')[:].astype(float).mean(0)
fit_dict['x_coords_outliers'] = self.x_array[probability_of_points < self.prob_threshold]
fit_dict['y_coords_outliers'] = self.y_array[probability_of_points < self.prob_threshold]
return fit_dict
coefs[tName] = Normal(tName,0,0.001,value=sp.rand()-0.5)
termList.append(d*coefs[tName])
# get individual edge probabilities
@deterministic(trace=False,plot=False)
def probs(termList=termList):
probs = 1./(1+sp.exp(-1*sum(termList)))
probs[sp.diag_indices_from(probs)]= 0
return(probs)
# define the outcome as
outcome = Bernoulli('outcome',probs,value=adjMat,observed=True)
return(locals())
if __name__ == '__main__':
# load the prison data
with open('prison.dat','r') as f:
rowList = list()
for l in f:
rowList.append([int(x) for x in l.strip().split(' ')])
adjMat = sp.array(rowList)
# make the model as an MCMC object
m = makeModel(adjMat)
mc = MCMC(m)
# estimate
mc.sample(30000,1000,50)
@observed(dtype=int, plot=False)
def zip(value=data, mu=mu, psi=psi):
""" Zero-inflated Poisson likelihood """
# Initialize likeihood
like = 0.0
# Loop over data
for x in value:
if not x:
# Zero values
like += np.log((1. - psi) + psi * np.exp(-mu))
else:
# Non-zero values
like += np.log(psi) + poisson_like(x, mu)
return like
if __name__ == "__main__":
from pymc import MCMC, Matplot
# Run model and plot posteriors
M = MCMC(locals())
M.sample(100000, 50000)
Matplot.plot(M)
def runMCMCmodel(args):
"""
Simulate the survey data and run the MCMC luminosity calibration model.
Parameters
----------
args - Command line arguments
"""
mcmcParams=args['mcmcString']
surveyParams=args['surveyString']
priorParams=args['priorsString']
maxIter=int(mcmcParams[0])
burnIter=int(mcmcParams[1])
thinFactor=int(mcmcParams[2])
if surveyParams[5] == 'Inf':
magLim = np.Inf
else:
magLim = float(surveyParams[5])
S=U.UniformDistributionSingleLuminosity(int(surveyParams[0]), float(surveyParams[1]),
float(surveyParams[2]), float(surveyParams[3]), float(surveyParams[4]),
surveyLimit=magLim)
#S.setRandomNumberSeed(53949896)
S.generateObservations()
lumCalModel=L.UniformSpaceDensityGaussianLFBook(S,float(surveyParams[1]), float(surveyParams[2]),
float(priorParams[0]), float(priorParams[1]), float(priorParams[2]), float(priorParams[3]))
class SurveyData(IsDescription):
"""
Class that holds the data model for the data from the simulated parallax survey. Intended for use
with the HDF5 files through the pytables package.
"""
trueParallaxes = Float64Col(S.numberOfStarsInSurvey)
absoluteMagnitudes = Float64Col(S.numberOfStarsInSurvey)
apparentMagnitudes = Float64Col(S.numberOfStarsInSurvey)
parallaxErrors = Float64Col(S.numberOfStarsInSurvey)
magnitudeErrors = Float64Col(S.numberOfStarsInSurvey)
observedParallaxes = Float64Col(S.numberOfStarsInSurvey)
observedMagnitudes = Float64Col(S.numberOfStarsInSurvey)
baseName="LumCalSimSurvey-{0}".format(S.numberOfStars)+"-{0}".format(S.minParallax)
baseName=baseName+"-{0}".format(S.maxParallax)+"-{0}".format(S.meanAbsoluteMagnitude)
baseName=baseName+"-{0}".format(S.varianceAbsoluteMagnitude)
h5file = openFile(baseName+".h5", mode = "w", title = "Simulated Survey")
group = h5file.createGroup("/", 'survey', 'Survey parameters, data, and MCMC parameters')
parameterTable = h5file.createTable(group, 'parameters', SurveyParameters, "Survey parameters")
dataTable = h5file.createTable(group, 'data', SurveyData, "Survey data")
mcmcTable = h5file.createTable(group, 'mcmc', McmcParameters, "MCMC parameters")
surveyParams = parameterTable.row
surveyParams['kind']=S.__class__.__name__
surveyParams['numberOfStars']=S.numberOfStars
surveyParams['minParallax']=S.minParallax
surveyParams['maxParallax']=S.maxParallax
surveyParams['meanAbsoluteMagnitude']=S.meanAbsoluteMagnitude
surveyParams['varianceAbsoluteMagnitude']=S.varianceAbsoluteMagnitude
surveyParams['parallaxErrorNormalizationMagnitude']=S.parallaxErrorNormalizationMagnitude
surveyParams['parallaxErrorSlope']=S.parallaxErrorSlope
surveyParams['parallaxErrorCalibrationFloor']=S.parallaxErrorCalibrationFloor
surveyParams['magnitudeErrorNormalizationMagnitude']=S.magnitudeErrorNormalizationMagnitude
surveyParams['magnitudeErrorSlope']=S.magnitudeErrorSlope
surveyParams['magnitudeErrorCalibrationFloor']=S.magnitudeErrorCalibrationFloor
surveyParams['apparentMagnitudeLimit']=S.apparentMagnitudeLimit
surveyParams['numberOfStarsInSurvey']=S.numberOfStarsInSurvey
surveyParams.append()
parameterTable.flush()
surveyData = dataTable.row
surveyData['trueParallaxes']=S.trueParallaxes
surveyData['absoluteMagnitudes']=S.absoluteMagnitudes
surveyData['apparentMagnitudes']=S.apparentMagnitudes
surveyData['parallaxErrors']=S.parallaxErrors
surveyData['magnitudeErrors']=S.magnitudeErrors
surveyData['observedParallaxes']=S.observedParallaxes
surveyData['observedMagnitudes']=S.observedMagnitudes
surveyData.append()
dataTable.flush()
mcmcParameters = mcmcTable.row
mcmcParameters['iterations']=maxIter
mcmcParameters['burnIn']=burnIter
mcmcParameters['thin']=thinFactor
mcmcParameters['minMeanAbsoluteMagnitude']=float(priorParams[0])
mcmcParameters['maxMeanAbsoluteMagnitude']=float(priorParams[1])
mcmcParameters['priorTau']="OneOverX"
mcmcParameters['tauLow']=float(priorParams[2])
mcmcParameters['tauHigh']=float(priorParams[3])
mcmcParameters.append()
dataTable.flush()
h5file.close()
# Run MCMC and store in HDF5 database
baseName="LumCalResults-{0}".format(S.numberOfStars)+"-{0}".format(S.minParallax)
baseName=baseName+"-{0}".format(S.maxParallax)+"-{0}".format(S.meanAbsoluteMagnitude)
baseName=baseName+"-{0}".format(S.varianceAbsoluteMagnitude)
M=MCMC(lumCalModel.pyMCModel, db='hdf5', dbname=baseName+".h5", dbmode='w', dbcomplevel=9,
dbcomplib='bzip2')
M.use_step_method(Metropolis, M.priorParallaxes)
M.use_step_method(Metropolis, M.priorAbsoluteMagnitudes)
start=now()
M.sample(iter=maxIter, burn=burnIter, thin=thinFactor)
finish=now()
print "Elapsed time in seconds: %f" % (finish-start)
M.db.close()
# coding: utf-8
"""
pyMCによるモンテカルロサンプリングの練習
"""
import numpy as np
import matplotlib.pyplot as plt
from pymc import Normal, Beta, Bernoulli, deterministic, MCMC, Matplot
real_sigma = 0.1 ** 2 # 分散の真の分布
real_mu = 1.0 # 平均の真の分布
x_sample = np.random.normal(real_mu, real_sigma, 5)
mu = Normal('mu', 0, 1.0/real_sigma)
x = Normal('x', mu=mu, tau=1.0 /real_sigma, value=x_sample, observed=True)
M = MCMC(input=[mu, x])
M.sample(iter=10000)
Matplot.plot(M)
def test_regression_155():
"""thin > iter"""
M = MCMC(disaster_model, db='ram')
M.sample(10,0,100, progress_bar=0)
db = pymc.database.pickle.load(fname)
A = MCMC(model, db=db)
if args.type != 'TOYSAVE':
with open(fmetaname,'r') as f:
logp, errors_b, errors_x = pickle.load(f)
print "Loaded previous model from %s" % fname
except (IOError, TypeError) as e:
# run new simulation
if fname:
A = MCMC(model, db='pickle', dbname=fname)
else:
A = MCMC(model)
logp = []
errors_b = []
errors_x = []
print "Created new model at %s" % fname
A.sample(iter=100)
if args.type == 'GRID':
A, logp, errors_b, errors_x = sample(model,A,fmetaname,iters=iters, \
logp=logp,errors_b=errors_b,errors_x=errors_x)
plot(logp, errors_b, errors_x)
elif args.type == 'TOY':
A, logp, errors_b, errors_x = sample_toy(model,A,fmetaname,iters=iters, \
logp=logp,errors_b=errors_b,errors_x=errors_x)
plot(logp, errors_b, errors_x)
elif args.type == 'TOYSAVE':
A, logp, errors_b, errors_x = sample_toy_save(model,A,iters=iters)
# The mu and tau are in log units; to get to log units,
# do the following
# (has mean around 1e2, with a variance of 9 logs in base 10)
mean_b10 = 2
var_b10 = 9
print "Setting mean (base 10) to %f, variance (base 10) to %f" % (mean_b10, var_b10)
# The lognormal variable
k = Lognormal('k', mu=np.log(10 ** mean_b10),
tau=1./(np.log(10) * np.log(10 ** var_b10)))
# Sample it
m = MCMC(Model([k]))
m.sample(iter=50000)
ion()
# Plot the distribution in base e
figure()
y = log(m.trace('k')[:])
y10 = log10(m.trace('k')[:])
hist(y, bins=100)
print
print "Mean, base e: %f; Variance, base e: %f" % (mean(y), var(y))
# Plot the distribution in base 10
figure()
hist(y10, bins=100)
print "Mean, base 10: %f; Variance, base 10: %f" % (mean(y10), var(y10))
def test_regression_155():
"""thin > iter"""
M = MCMC(DisasterModel, db='ram')
M.sample(10,0,100)
# Ph21 Set 5 # Aritra Biswas # coin_mcmc.py # Run MCMC on coin_model.py import coin_model from pymc import MCMC from pymc.Matplot import plot M = MCMC(coin_model) M.sample(iter = 10000, burn = 0, thin = 1) print plot(M) M.pheads.summary()
def p_segment_mcmc(
filtered_sig,
annots,
fs,
step=2,
iter_count=4000,
burn_count=2000,
max_hermit_level=4,
savefig_path=None,
figID=None,
):
'''Detect and returns P wave detection results.
Note:
* Input is a segment contains only P wave.
* This function returns None when detection fails.
'''
filtered_sig = np.array(filtered_sig)
# Normalization
max_val = np.max(filtered_sig)
min_val = np.min(filtered_sig)
if (max_val - min_val) > 1e-6:
filtered_sig = (filtered_sig - min_val) / (max_val - min_val)
raw_sig = filtered_sig
sig_seg = raw_sig
p_model = P_model_Gaussian.MakeModel(sig_seg,
annots,
max_hermit_level=max_hermit_level)
M = MCMC(p_model)
M.sample(iter=iter_count, burn=burn_count, thin=10)
# retrieve parameters
hermit_coefs = list()
for h_ind in xrange(0, P_model_Gaussian.HermitFunction_max_level):
hermit_value = np.mean(M.trace('hc%d' % h_ind)[:])
hermit_coefs.append(hermit_value)
fitting_curve = np.zeros(len(sig_seg), )
for level, coef in zip(xrange(0, max_hermit_level), hermit_coefs):
fitting_curve += P_model_Gaussian.HermitFunction(level,
len(sig_seg)) * coef
# Gaussian
pos_ponset = None
pos_p = None
pos_poffset = None
for pos, label in annots:
if label == 'Ponset':
pos_ponset = pos
elif label == 'P':
pos_p = pos
elif label == 'Poffset':
pos_poffset = pos
gaussian_curve = np.zeros(len(sig_seg), )
common_length = len(sig_seg)
g_amp = np.mean(M.trace('g_amp')[:])
g_sigma = np.mean(M.trace('g_sigma')[:])
g_dc = np.mean(M.trace('dc')[:])
gaussian_curve = P_model_Gaussian.GetGaussianPwave(
len(sig_seg) * 2, g_amp, g_sigma / 3, g_dc)
gaussian_segment = gaussian_curve[common_length - pos_p:2 * common_length -
pos_p]
baseline_curve = fitting_curve + g_dc
baseline_curve = baseline_curve.tolist()
fitting_curve += gaussian_segment
# Compute results
results = dict(P=pos_p)
meet_threshold = 0.07
for ind in xrange(0, len(fitting_curve)):
if ('Ponset' not in results
and abs(fitting_curve[ind] - baseline_curve[ind]) >=
meet_threshold):
results['Ponset'] = ind
elif ('Ponset' in results and
abs(fitting_curve[ind] - baseline_curve[ind]) >= meet_threshold):
results['Poffset'] = ind
# If not found
if 'Ponset' not in results:
results['Ponset'] = pos_ponset
results['Poffset'] = pos_poffset
elif 'Poffset' not in results:
results['Poffset'] = pos_poffset
return results
