def full_analysis(galaxy=None, opt='kin'): galaxies = np.array(['ngc3557', 'ic1459', 'ic1531', 'ic4296', 'ngc0612', 'ngc1399', 'ngc3100', 'ngc7075', 'pks0718-34', 'eso443-g024']) # an inital guess from quick internet search of redshift. z_gals = [0.01, 0.005, 0.025, 0.01, 0.028, 0.005, 0.01, 0.02, 0.03, 0.015] gal=6 if galaxy is None: galaxy = galaxies[gal] else: gal = np.where(galaxies == galaxy)[0][0] print galaxy z = z_gals[gal] discard = 0 targetSN = 30. set_range = [4200, 10000] binning_spaxels(galaxy, discard=discard, targetSN=targetSN, opt=opt, auto_override=True) find_template(galaxy, z=z, discard=discard, set_range=set_range) mcmc(galaxy, z=z, discard=discard, set_range=set_range)
def efficiency():
"""
Efficiency as the number of clusters growing.
"""
N, M, Psi, GT = load_dataset()
# inject confusions
Ncs = [10, 100, 1000, 10000]
mcmc_params = {'N_iter': 10, 'burnin': 1, 'thin': 2, 'FV': 3}
res = {'mv': [],
'mv std': [],
'em': [],
'em std': [],
'mcmc': [],
'mcmc std': [],
'f_mcmc': [],
'f_mcmc std': [],
'number of objects with confusions': Ncs}
for Nc in Ncs:
times = [[], [], [], []]
for run in range(n_runs):
GT_G, Cl, cPsi = confuse(Psi, 0.8, GT, Nc)
start = time.time()
mv_p = majority_voting(cPsi)
times[0].append(time.time() - start)
start = time.time()
em_A, em_p = expectation_maximization(N, M, cPsi)
times[1].append(time.time() - start)
start = time.time()
mcmc_A, mcmc_p = mcmc(N, M, cPsi, mcmc_params)
times[2].append(time.time() - start)
start = time.time()
f_mcmc_A, f_mcmc_p, f_mcmc_G = f_mcmc(N, M, cPsi, Cl, mcmc_params)
times[3].append(time.time() - start)
res['mv'].append(np.average(times[0]))
res['em'].append(np.average(times[1]))
res['mcmc'].append(np.average(times[2]))
res['f_mcmc'].append(np.average(times[3]))
res['mv std'].append(np.std(times[0]))
res['em std'].append(np.std(times[1]))
res['mcmc std'].append(np.std(times[2]))
res['f_mcmc std'].append(np.std(times[3]))
print('{}\tmv: {:1.4f}\tem: {:1.4f}\tmcmc: {:1.4f}\tf_mcmc: {:1.4f}'.format(Nc,
np.average(times[0]),
np.average(times[1]),
np.average(times[2]),
np.average(times[3])
)
)
pd.DataFrame(res).to_csv(work_dir + 'flight_efficiency.csv', index=False)
def run_mcmc(model_prior_params, data_prior_params, S, N, sensitivity_x,
sensitivity_y, epsilon):
Z, sensitivity = privatize_suff_stats(S, sensitivity_x, sensitivity_y,
epsilon)
theta, sigma_squared = mcmc(model_prior_params, data_prior_params, N,
epsilon, Z, sensitivity, 2000)
return theta, sigma_squared
def execute(saveloc, numit, mag=False):
rundir = "/home/esp01/events/dynamics/transits/"
time, flux, error = loadtr(rundir + saveloc + "/data.txt")
if mag == True:
flux = 10**(-flux / 2.5)
error = 100**(-error / 5.) - 1
print("Converting to relative flux...", error.shape)
trdata = time, flux, error
init, pmin, pmax, step = loadparams(rundir + saveloc + "/params.txt")
numparams = []
numparams.append(0)
numparams.append(len(init))
allparams, best, numaccept = mcmc.mcmc(flux, time, init, pmin, pmax, step,
numit, error, numparams, 1,
[quadlimb], [time], 0)
save(saveloc + "/output.npz", allparams, best, numaccept, trdata)
print(np.float(numaccept) / numit * 100)
return allparams, best, numaccept, trdata
def run(self, *args, **kwargs):
h = h5py.File('{}/EXTRACT.hd5'.format(self.genParams['outDir']))
sources = list(h.keys())[:]
h.close()
status = ['1band', '8band']
print(sources)
sources = []
for s in tqdm(sources):
h = h5py.File('{}/EXTRACT.hd5'.format(self.genParams['outDir']))
n_surveys = len(list(h['{}/APflux'.format(s)].keys()))
data = np.zeros((n_surveys),
dtype=[('name', 'U50'), ('nu', 'f8'), ('S', 'f8'),
('Se', 'f8')])
for _, survey in enumerate(list(h['{}/APflux'.format(s)].keys())):
data[_]['name'] = survey
data[_]['nu'] = h['{}/APflux/{}'.format(s, survey)][0]
data[_]['S'] = h['{}/APflux/{}'.format(s, survey)][1]
data[_]['Se'] = h['{}/APflux/{}'.format(s, survey)][2]
h.close()
nu = data['nu']
S = data['S']
Se = data['Se']
# fig = plt.figure()
# ax = plt.subplot()
# ax.scatter(nu/1e9,S)
# ax.set_xlabel(r'$\nu$ [GHz]')
# ax.set_xlabel(r'$S_\nu$ [Jy]')
# TRIGGER
sys.path.insert(0, '/Volumes/TJRennie/github/mcmc')
import mcmc, emission, tools
from config_mcmc import settings
del sys.path[0]
custom = {
'source_name': s,
'result_dir': self.genParams['outDir'],
'plot_dir': self.genParams['outDir'],
}
settings['source_name'] = s
settings['plotting']['resultdir'] = self.genParams['outDir']
settings['plotting']['plotdir'] = self.genParams['outDir']
# settings['components']['ame'] = 0
# CORR = np.identity(len(nu))
# Se_ = np.asarray(Se)[:,np.newaxis]
# CORR = Se_*CORR*Se_.T
# # try:
# COMAP_corr = np.load('COMAP_correlation.npy')
c_true = np.zeros(len(nu)).astype(bool)
for _ in range(len(nu)):
if data['name'][_][:5] == 'COMAP':
c_true[_] = True
# a = COMAP_corr*np.mean(np.diag(CORR[c_true][:,c_true]))
# CORR[4:12,4:12] = a
mask1 = np.all([
nu != 100e9,
nu != 217e9,
nu != 4997e9,
nu != 26.5,
nu != 27.5,
nu != 28.5,
nu != 30.5,
nu != 31.5,
nu != 32.5,
nu != 33.5,
],
axis=0)
#np.all([nu<1e11,S>1e3],axis=0)==False,
use = np.all([nu < 4e12, np.isfinite(S), S > 1e-1, mask1], axis=0)
# try:
mcmc_data, mcmc_model, mcmc_settings = mcmc.mcmc(
nu[use] / 1e9,
S[use],
Se[use],
beam=np.radians(60. / 60.)**2,
custom_settings=settings,
excluded=[])
# except:
# continue
def model(x):
return mcmc_model['sed_model'](x, np.radians(60. / 60.)**2 *
np.pi / 4,
mcmc_model['sed_params'])
fig = plt.figure()
ax = plt.subplot()
argn = 0 # first argument number
style_rep = ['--', '-.', ':']
for i in mcmc_model['components']['n_components']:
component = mcmc_model['components']['function'][i]
comp_nameparams = mcmc_model['components']['name_params'][i]
comp_nparams = mcmc_model['components']['n_params'][i]
args_to_pass = mcmc_model['sed_params'][argn:argn +
comp_nparams]
component_flux = component(
10**np.arange(-1, 4.01, 0.01),
np.radians(60. / 60.)**2 * np.pi / 4,
*args_to_pass) # call each function and add up the flux
linestyle_to_use = np.mod(i, len(style_rep))
ax.plot(10**np.arange(-1, 4.01, 0.01),
component_flux,
color="black",
linestyle=style_rep[linestyle_to_use],
alpha=0.5)
argn += comp_nparams
ax.plot(10**np.arange(-1, 4.01, 0.01),
model(10**np.arange(-1, 4.01, 0.01)),
c='red')
ax.errorbar(nu[use] / 1e9,
S[use],
yerr=Se[use],
marker='o',
linestyle='none',
ms=4,
c='black')
ax.errorbar(nu[use == False] / 1e9,
S[use == False],
yerr=Se[use == False],
marker='o',
fmt='o',
mfc='white',
c='black',
ms=4)
ax.set_xscale("log", nonpositive='clip')
ax.set_yscale("log", nonpositive='clip')
ax.set_title(s)
ax.set_xlabel(r'$nu$ [GHz]')
ax.set_ylabel(r'$S_\nu$ [Jy]')
try:
ax.set_ylim(
10.**float((np.log10(
np.nanmin(model(10**np.arange(-1, 4.01, 0.01)))) //
1.)),
10.**float((np.log10(
np.nanmax(model(10**np.arange(-1, 4.01, 0.01)))) // 1.
+ 1.)))
except:
ax.set_ylim(10**-2, 10**3)
ax.set_xlim((10**0, 10**4.5))
Inset_axes = ax.inset_axes([0.08, 0.65, 0.4, 0.3])
Inset_axes.plot(np.arange(26, 34.01, 0.02),
model(np.arange(26, 34.01, 0.02)),
c='red')
# print(c_true)
Inset_axes.errorbar(nu[c_true] / 1e9,
S[c_true],
yerr=Se[c_true],
marker='o',
linestyle='none',
ms=4,
c='black')
Inset_axes.set_ylim(
(np.nanmin(S[c_true]) - 0.15, np.nanmax(S[c_true]) + 0.15))
Inset_axes.set_xlim((25.5, 34.5))
outfilename = '{}/{}_SpectrumPlots_model_ONE_fit.png'.format(
self.genParams['outDir'], s)
plt.savefig(outfilename)
plt.close()
# chi2 = mcmc_model['sed_chi_squared']
# self.ver_print('Chi2: {}'.format(chi2). ver_no=2)
h = h5py.File('{}/EXTRACT.hd5'.format(self.genParams['outDir']),
'a')
# if '{}/rokeFit/AME;FreeFree;ThDust(7)_model_ONE/redChi2'.format(s) in h:
# del h['{}/rokeFit/AME;FreeFree;ThDust(7)_model_ONE/redChi2'.format(s)]
# h['{}/rokeFit/AME;FreeFree;ThDust(7)_model_ONE/redChi2'.format(s)] = chi2Red
if '{}/rokeFit/AME;FreeFree;ThDust(7)_model_ONE/params'.format(
s) in h:
del h['{}/rokeFit/AME;FreeFree;ThDust(7)_model_ONE/params'.
format(s)]
h['{}/rokeFit/AME;FreeFree;ThDust(7)_model_ONE/params'.format(
s)] = mcmc_model['sed_params']
h.close()
# except:
# continue
return True, None
mu_th = np.array([0, 0, 0]) # mean & std for prior
cov_th = np.power(np.eye(dim) * 0.3, 2)
mu_eps = 0 # mean & std for likelihood
cov_eps = np.power(0.2, 2)
elif case == 4: # system id: theta = prefactors in the PDE to be identified
niter = 6000 # num of MCMC interations
dim = 3 # model dimensions
mu_th = np.zeros(dim) # mean & std for prior
cov_th = np.ones(dim) * 0.3 # Laplace scaling factors
mu_eps = 0 # mean & std for likelihood
cov_eps = np.power(0.2, 2)
isSparse = True
data = genData(case, noise_level) # data
modelFun = lambda th: testFun(th, case) # model
# Model initialization
model = Model(modelFun, mu_th, cov_th, mu_eps, cov_eps)
test = mcmc(niter, data, model, isSparse)
return test
if __name__ == "__main__":
case = 1
noise_level = 0.02
test = testCase(case, noise_level)
print('Test case: ', case)
print('Model attributes: ', test.model.attrList())
print('Exact data sin(2) =', test.model.modelFun(2))
print('Noisy data sin(2) ≈', test.data)
def idea3(G):
T, samples = mcmc(G, 100000)
return T
def accuracy():
"""
Vary the confusion probability on real data.
"""
N, M, Psi, GT = load_dataset()
# inject confusions
conf_probs = [0.0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4]
mcmc_params = {'N_iter': 10, 'burnin': 1, 'thin': 2, 'FV': 3}
res = {'mv':[], 'em': [], 'mcmc': [], 'f_mcmc':[], 'confusion probability': conf_probs,
'mv_std': [], 'em_std': [], 'mcmc_std': [], 'f_mcmc_std': [], 'G': [], 'G_std': []}
for conf_prob in conf_probs:
runs = [[], [], [], [], []]
for run in range(n_runs):
GT_G, Cl, cPsi = confuse(Psi, 1-conf_prob, GT)
start = time.time()
mv_p = majority_voting(cPsi)
mv_t = time.time() - start
start = time.time()
em_A, em_p = expectation_maximization(N, M, cPsi)
em_t = time.time() - start
start = time.time()
mcmc_A, mcmc_p = mcmc(N, M, cPsi, mcmc_params)
mcmc_t = time.time() - start
start = time.time()
f_mcmc_A, f_mcmc_p, f_mcmc_G = f_mcmc(N, M, cPsi, Cl, mcmc_params)
f_mcmc_t = time.time() - start
mv_accu = np.average([mv_p[obj][GT[obj]] for obj in GT.keys()])
em_accu = np.average([em_p[obj][GT[obj]] for obj in GT.keys()])
mcmc_accu = np.average([mcmc_p[obj][GT[obj]] for obj in GT.keys()])
f_mcmc_accu = np.average([f_mcmc_p[obj][GT[obj]] for obj in GT.keys()])
runs[0].append(mv_accu)
runs[1].append(em_accu)
runs[2].append(mcmc_accu)
runs[3].append(f_mcmc_accu)
runs[4].append(accu_G(f_mcmc_G, GT_G))
res['mv'].append(np.average(runs[0]))
res['em'].append(np.average(runs[1]))
res['mcmc'].append(np.average(runs[2]))
res['f_mcmc'].append(np.average(runs[3]))
res['G'].append(np.average(runs[4]))
res['mv_std'].append(np.std(runs[0]))
res['em_std'].append(np.std(runs[1]))
res['mcmc_std'].append(np.std(runs[2]))
res['f_mcmc_std'].append(np.std(runs[3]))
res['G_std'].append(np.average(runs[4]))
print('{}\tmv: {:1.4f}\tem: {:1.4f}\tmcmc: {:1.4f}\tf_mcmc: {:1.4f}'.format(conf_prob,
mv_accu,
em_accu,
mcmc_accu,
f_mcmc_accu
)
)
pd.DataFrame(res).to_csv(work_dir + 'flight_accuracy.csv', index=False)
T = 5.0
def get_probs(samples, expression):
length = (len(expression) / WINDOWS) + 1
counts = [0] * length
for sample in samples:
for ratchet in sample:
counts[ratchet] += 1
max_expression = max(expression)
for i in xrange(len(counts)):
counts[i] = (counts[i] * max_expression) / float(len(samples))
return counts
data = open(sys.argv[1], 'r')
c = 0
with open(sys.argv[1]) as fp:
for line in fp:
start, end = line.split()[1:3]
if int(end) - int(start) < WINDOWS * 7: continue
expression = [int(i) for i in line.strip().split('\t')[-1].split(",")]
c += 1
if line.split('\t')[-2] == "-":
expression.reverse()
samples = mcmc(expression, WINDOWS, T)
print line.strip() + '\t' + ','.join(
map(str, get_probs(samples, expression)))
'non-stunted']
meta_prefit['mortality, 12-24, stunted'] = P['mortality']['12-24']['stunted']
meta_prefit['mortality, 24-72, non-stunted'] = P['mortality']['24-72'][
'non-stunted']
meta_prefit['mortality, 24-72, stunted'] = P['mortality']['24-72']['stunted']
nwalkers = 20
nsteps = 50
nburn = 10
storage = "mcmc"
mc = mcmc(meta_prefit,
C0,
P,
D,
t_steps,
pops,
nwalkers=20,
nsteps=50,
nburn=10,
storage="mcmc")
P['mortality']['0-1']['non-stunted'] = np.mean(mc[:, 0])
P['mortality']['0-1']['stunted'] = np.mean(mc[:, 1])
P['mortality']['1-6']['non-stunted'] = np.mean(mc[:, 2])
P['mortality']['1-6']['stunted'] = np.mean(mc[:, 3])
P['mortality']['6-12']['non-stunted'] = np.mean(mc[:, 4])
P['mortality']['6-12']['stunted'] = np.mean(mc[:, 5])
P['mortality']['12-24']['non-stunted'] = np.mean(mc[:, 6])
P['mortality']['12-24']['stunted'] = np.mean(mc[:, 7])
P['mortality']['24-72']['non-stunted'] = np.mean(mc[:, 8])
P['mortality']['24-72']['stunted'] = np.mean(mc[:, 9])
def run(self, resol=30, *args, **kwargs):
h = h5py.File('{}/EXTRACT.hd5'.format(self.genParams['outDir']))
sources = list(h.keys())[:]
h.close()
status = ['1band', '8band']
print(sources)
# sources = ['G030.750-00.020']
for s in tqdm(sources[:]):
h = h5py.File('{}/EXTRACT.hd5'.format(self.genParams['outDir']))
n_surveys = len(list(h['{}/APflux'.format(s)].keys()))
data = np.zeros((n_surveys),
dtype=[('name', 'U50'), ('nu', 'f8'), ('S', 'f8'),
('Se', 'f8')])
for _, survey in enumerate(list(h['{}/APflux'.format(s)].keys())):
data[_]['name'] = survey
data[_]['nu'] = h['{}/APflux/{}'.format(s, survey)][0]
data[_]['S'] = h['{}/APflux/{}'.format(s, survey)][1]
data[_]['Se'] = h['{}/APflux/{}'.format(s, survey)][2]
h.close()
nu = data['nu']
S = data['S']
Se = data['Se']
# TRIGGER
sys.path.insert(0, '/Volumes/TJRennie/github/mcmcCorr')
import mcmc, emission, tools
from config_mcmc import settings
del sys.path[0]
custom = {
'source_name': s,
'result_dir': self.genParams['outDir'],
'plot_dir': self.genParams['outDir'],
}
settings['source_name'] = s
settings['plotting']['resultdir'] = self.genParams['outDir']
settings['plotting']['plotdir'] = self.genParams['outDir']
#
settings['components']['ame'] = 0
settings['components']['synchrotron'] = 1
settings['components']['freefree'] = 0
runname = 'FINAL_SNRfit2'
CORR = np.identity(len(nu))
Se_ = np.asarray(Se)[:, np.newaxis]
CORR = Se_ * CORR * Se_.T
try:
COMAP_corr = np.load('COMAP_correlation.npy')
c_true = np.zeros(len(nu)).astype(bool)
for _ in range(len(nu)):
if data['name'][_][:5] == 'COMAP':
c_true[_] = True
a = COMAP_corr * np.mean(np.diag(CORR[c_true][:, c_true]))
CORR[c_true][:, c_true] = a
def chk(a, b, c):
return abs(a - b) > c
mask1 = np.all(
[nu != 100e9, nu != 217e9, nu != 4997e9, nu != 10e9],
axis=0)
#np.all([nu<1e11,S>1e3],axis=0)==False,
use = np.all(
[nu < 400e9, np.isfinite(S), S > 1e-3, mask1], axis=0)
# print(nu[mask1])
# print('!!!!!')
# for n in use:
# if n:
# print(nu[n]/1e9,S[n],Se[n,n])
# fig = plt.figure()
# ax = plt.subplot()
# # ax.scatter(nu[mask1]/1e9,S[mask1],c='red')
# ax.scatter(nu[use]/1e9,S[use],c='red')
# ax.scatter(nu[use==False]/1e9,S[use==False],c='blue')
# ax.set_xlabel(r'$\nu$ [GHz]')
# ax.set_xlabel(r'$S_\nu$ [Jy]')
# plt.loglog()
# plt.show()
try:
mcmc_data, mcmc_model, mcmc_settings = mcmc.mcmc(
nu[use] / 1e9,
S[use],
CORR[use][:, use],
beam=np.radians(resol / 60.)**2,
custom_settings=settings,
excluded=[])
except:
pass
def model(x):
return mcmc_model['sed_model'](x,
np.radians(resol / 60)**2,
mcmc_model['sed_params'])
fig = plt.figure()
ax = plt.subplot()
argn = 0 # first argument number
style_rep = ['--', '-.', ':']
paramDict = {}
for i in mcmc_model['components']['n_components']:
component = mcmc_model['components']['function'][i]
comp_nameparams = mcmc_model['components']['name_params'][
i]
comp_nparams = mcmc_model['components']['n_params'][i]
args_to_pass = mcmc_model['sed_params'][argn:argn +
comp_nparams]
component_flux = component(
10**np.arange(-1, 4.01, 0.01),
np.radians(resol / 60)**2, *args_to_pass
) # call each function and add up the flux
linestyle_to_use = np.mod(i, len(style_rep))
ax.plot(10**np.arange(-1, 4.01, 0.01),
component_flux,
color="black",
linestyle=style_rep[linestyle_to_use],
alpha=0.5)
argn += comp_nparams
fname = str(component).split(' ')[1]
paramDict[fname] = {}
paramDict[fname]['1'] = {}
for _ in range(len(args_to_pass)):
p = comp_nameparams[_]
v = args_to_pass[_]
paramDict[fname]['1'][p] = v
ax.plot(10**np.arange(-1, 4.01, 0.01),
model(10**np.arange(-1, 4.01, 0.01)),
c='red')
# ax.errorbar(nu[:]/1e9,S[:],yerr=Se[:],marker='x',linestyle='none',ms=4,c='black')
ax.errorbar(nu[use] / 1e9,
S[use],
yerr=Se[use],
marker='o',
linestyle='none',
ms=4,
c='black')
ax.errorbar(nu[use == False] / 1e9,
S[use == False],
yerr=Se[use == False],
marker='o',
fmt='o',
mfc='white',
c='black',
ms=4)
ax.set_xscale("log", nonpositive='clip')
ax.set_yscale("log", nonpositive='clip')
ax.set_title(s)
ax.set_xlabel(r'$nu$ [GHz]')
ax.set_ylabel(r'$S_\nu$ [Jy]')
try:
ax.set_ylim(
10.**float((np.log10(
np.nanmin(model(10**np.arange(-1, 4.01, 0.01)))) //
1.)),
10.**float((np.log10(
np.nanmax(model(10**np.arange(-1, 4.01, 0.01)))) //
1. + 1.)))
except:
ax.set_ylim(10**-2, 10**3)
ax.set_xlim((10**0, 10**4.5))
Inset_axes = ax.inset_axes([0.08, 0.65, 0.4, 0.3])
Inset_axes.plot(np.arange(26, 34.01, 0.02),
model(np.arange(26, 34.01, 0.02)),
c='red')
Inset_axes.errorbar(nu[c_true] / 1e9,
S[c_true],
yerr=Se[c_true],
marker='o',
linestyle='none',
ms=4,
c='black')
Inset_axes.set_ylim(
(np.nanmin(S[c_true]) - 0.15, np.nanmax(S[c_true]) + 0.15))
Inset_axes.set_xlim((25.5, 34.5))
outfilename = '{}/{}_SpectrumPlots_{}_fit.png'.format(
self.genParams['outDir'], s, runname)
# plt.show()
plt.savefig(outfilename)
plt.close()
# ndof = np.nansum(use) - len(mcmc_model['sed_params'])
# print('ndof: ',ndof)
# chi2 = np.nansum((S[use]-model(nu[use]))**2/(np.diag(CORR)[use]), axis=0)
# print('Chi2: ',chi2)
h = h5py.File('{}/EXTRACT.hd5'.format(
self.genParams['outDir']),
mode='a')
# if '{}/rokeFit/AME;FreeFree;ThDust(7)_{}/redChi2'.format(s,runname) in h:
# del h['{}/rokeFit/AME;FreeFree;ThDust(7)_{}/redChi2'.format(s,runname)]
# h['{}/rokeFit/AME;FreeFree;ThDust(7)_{}/redChi2'.format(s,runname)] = chi2
# if '{}/rokeFit/AME;FreeFree;ThDust(7)_{}/params'.format(s,runname) in h:
# del h['{}/rokeFit/AME;FreeFree;ThDust(7)_{}/params'.format(s,runname)]
# h['{}/rokeFit/AME;FreeFree;ThDust(7)_{}/params'.format(s,runname)] = mcmc_model['sed_params']
def pushDict(h5file, obj):
if isinstance(obj, dict) and isinstance(
h5file, h5py.Group):
for key, value in obj.items():
if isinstance(value, dict):
if key in h5file:
del h5file[key]
if key not in h5file:
h5file.create_group(key)
pushDict(h5file[key], value)
else:
if key in h5file:
del h5file[key]
h5file[key] = value
if '{}/Model/{}'.format(s, runname) in h:
del h['{}/Model/{}'.format(s, runname)]
h.create_group('{}/Model/{}'.format(s, runname))
h.create_group('{}/Model/{}/model_dict'.format(s, runname))
pushDict(h['{}/Model/{}/model_dict'.format(s, runname)],
paramDict)
h['{}/Model/{}/fitted'.format(s, runname)] = use
except:
pass
h.close()
return True, None
def main():
"""
Multi-Core Markov-Chain Monte Carlo (MC cubed)
This code calls MCMC to work under an MPI multiprocessor protocol or
single-thread mode. When using MPI it will launch one CPU per MCMC chain
to work in parallel.
Parameters:
-----------
cfile: String
Filename of a configuration file.
"""
# Parse the config file from the command line:
cparser = argparse.ArgumentParser(description=__doc__, add_help=False,
formatter_class=argparse.RawDescriptionHelpFormatter)
# Add config file option:
cparser.add_argument("-c", "--config_file",
help="Configuration file", metavar="FILE")
# Remaining_argv contains all other command-line-arguments:
args, remaining_argv = cparser.parse_known_args()
# Take configuration file from command-line:
cfile = args.config_file
# Incorrect configuration file name:
if cfile is not None and not os.path.isfile(cfile):
mu.error("Configuration file: '{:s}' not found.".format(cfile))
if cfile:
config = ConfigParser.SafeConfigParser()
config.read([cfile])
defaults = dict(config.items("MCMC"))
else:
defaults = {}
# Parser for the MCMC arguments:
parser = argparse.ArgumentParser(parents=[cparser])
# MCMC Options:
group = parser.add_argument_group("MCMC General Options")
group.add_argument("-n", "--numit",
dest="numit",
help="Number of MCMC samples [default: %(default)s]",
type=eval, action="store", default=100)
group.add_argument("-x", "--nchains",
dest="nchains",
help="Number of chains [default: %(default)s]",
type=int, action="store", default=10)
group.add_argument("-w", "--walk",
dest="walk",
help="Random walk algorithm [default: %(default)s]",
type=str, action="store", default="demc",
choices=('demc', 'mrw'))
group.add_argument( "--wlikelihood",
dest="wlike",
help="Calculate the likelihood in a wavelet base "
"[default: %(default)s]",
type=eval, action="store", default=False)
group.add_argument( "--leastsq",
dest="leastsq",
help="Perform a least-square minimization before the "
"MCMC run [default: %(default)s]",
type=eval, action="store", default=False)
group.add_argument( "--chisq_scale",
dest="chisqscale",
help="Scale the data uncertainties such that the reduced "
"chi-squared = 1. [default: %(default)s]",
type=eval, action="store", default=False)
group.add_argument("-g", "--gelman_rubin",
dest="grtest",
help="Run Gelman-Rubin test [default: %(default)s]",
type=eval, action="store", default=False)
group.add_argument( "--grexit",
dest="grexit",
help="Exit the MCMC loop if the MCMC satisfies the GR "
"test two consecutive times [default: %(default)s]",
type=eval, action="store", default=False)
group.add_argument("-b", "--burnin",
help="Number of burn-in iterations (per chain) "
"[default: %(default)s]",
dest="burnin",
type=eval, action="store", default=0)
group.add_argument("-t", "--thinning",
dest="thinning",
help="Chains thinning factor (use every thinning-th "
"iteration) for GR test and plots [default: %(default)s]",
type=int, action="store", default=1)
group.add_argument( "--plots",
dest="plots",
help="If True plot parameter traces, pairwise posteriors, "
"and marginal posterior histograms [default: %(default)s]",
type=eval, action="store", default=False)
group.add_argument("-o", "--save_file",
dest="savefile",
help="Output filename to store the parameter posterior "
"distributions [default: %(default)s]",
type=str, action="store", default="output.npy")
group.add_argument( "--savemodel",
dest="savemodel",
help="Output filename to store the evaluated models "
"[default: %(default)s]",
type=str, action="store", default=None)
group.add_argument( "--mpi",
dest="mpi",
help="Run under MPI multiprocessing [default: "
"%(default)s]",
type=eval, action="store", default=False)
group.add_argument( "--resume",
dest="resume",
help="If True, resume a previous run (load output) "
"[default: %(default)s]",
type=eval, action="store", default=False)
group.add_argument( "--rms",
dest="rms",
help="If True, calculate the RMS of (data-bestmodel) "
"[default: %(default)s]",
type=eval, action="store", default=False)
group.add_argument( "--logfile",
dest="logfile",
help="Log file.",
action="store", default=None)
group.add_argument("-T", "--tracktime", dest="tractime", action="store_true")
# Fitting-parameter Options:
group = parser.add_argument_group("Fitting-function Options")
group.add_argument("-f", "--func",
dest="func",
help="List of strings with the function name, module "
"name, and path-to-module [required]",
type=mu.parray, action="store", default=None)
group.add_argument("-p", "--params",
dest="params",
help="Filename or list of initial-guess model-fitting "
"parameter [required]",
type=mu.parray, action="store", default=None)
group.add_argument("-m", "--pmin",
dest="pmin",
help="Filename or list of parameter lower boundaries "
"[default: -inf]",
type=mu.parray, action="store", default=None)
group.add_argument("-M", "--pmax",
dest="pmax",
help="Filename or list of parameter upper boundaries "
"[default: +inf]",
type=mu.parray, action="store", default=None)
group.add_argument("-s", "--stepsize",
dest="stepsize",
help="Filename or list with proposal jump scale "
"[default: 0.1*params]",
type=mu.parray, action="store", default=None)
group.add_argument("-i", "--indparams",
dest="indparams",
help="Filename or list with independent parameters for "
"func [default: None]",
type=mu.parray, action="store", default=[])
# Data Options:
group = parser.add_argument_group("Data Options")
group.add_argument("-d", "--data",
dest="data",
help="Filename or list of the data being fitted "
"[required]",
type=mu.parray, action="store", default=None)
group.add_argument("-u", "--uncertainties",
dest="uncert",
help="Filemane or list with the data uncertainties "
"[default: ones]",
type=mu.parray, action="store", default=None)
group.add_argument( "--prior",
dest="prior",
help="Filename or list with parameter prior estimates "
"[default: %(default)s]",
type=mu.parray, action="store", default=None)
group.add_argument( "--priorlow",
dest="priorlow",
help="Filename or list with prior lower uncertainties "
"[default: %(default)s]",
type=mu.parray, action="store", default=None)
group.add_argument( "--priorup",
dest="priorup",
help="Filename or list with prior upper uncertainties "
"[default: %(default)s]",
type=mu.parray, action="store", default=None)
# Set the defaults from the configuration file:
parser.set_defaults(**defaults)
# Set values from command line:
args2, unknown = parser.parse_known_args(remaining_argv)
# Unpack configuration-file/command-line arguments:
numit = args2.numit
nchains = args2.nchains
walk = args2.walk
wlike = args2.wlike
leastsq = args2.leastsq
chisqscale = args2.chisqscale
grtest = args2.grtest
grexit = args2.grexit
burnin = args2.burnin
thinning = args2.thinning
plots = args2.plots
savefile = args2.savefile
savemodel = args2.savemodel
mpi = args2.mpi
resume = args2.resume
tracktime = args2.tractime
logfile = args2.logfile
rms = args2.rms
func = args2.func
params = args2.params
pmin = args2.pmin
pmax = args2.pmax
stepsize = args2.stepsize
indparams = args2.indparams
data = args2.data
uncert = args2.uncert
prior = args2.prior
priorup = args2.priorup
priorlow = args2.priorlow
nprocs = nchains
# Open a log FILE if requested:
if logfile is not None:
log = open(logfile, "w")
else:
log = None
# Handle arguments:
if params is None:
mu.error("'params' is a required argument.", log)
elif isinstance(params[0], str):
# If params is a filename, unpack:
if not os.path.isfile(params[0]):
mu.error("params file '{:s}' not found.".format(params[0]), log)
array = mu.loadascii(params[0])
# Array size:
ninfo, ndata = np.shape(array)
if ninfo == 7: # The priors
prior = array[4]
priorlow = array[5]
priorup = array[6]
if ninfo >= 4: # The stepsize
stepsize = array[3]
if ninfo >= 2: # The boundaries
pmin = array[1]
pmax = array[2]
params = array[0] # The initial guess
# Check for pmin and pmax files if not read before:
if pmin is not None and isinstance(pmin[0], str):
if not os.path.isfile(pmin[0]):
mu.error("pmin file '{:s}' not found.".format(pmin[0]), log)
pmin = mu.loadascii(pmin[0])[0]
if pmax is not None and isinstance(pmax[0], str):
if not os.path.isfile(pmax[0]):
mu.error("pmax file '{:s}' not found.".format(pmax[0]), log)
pmax = mu.loadascii(pmax[0])[0]
# Stepsize:
if stepsize is not None and isinstance(stepsize[0], str):
if not os.path.isfile(stepsize[0]):
mu.error("stepsize file '{:s}' not found.".format(stepsize[0]), log)
stepsize = mu.loadascii(stepsize[0])[0]
# Priors:
if prior is not None and isinstance(prior[0], str):
if not os.path.isfile(prior[0]):
mu.error("prior file '{:s}' not found.".format(prior[0]), log)
prior = mu.loadascii(prior [0])[0]
if priorlow is not None and isinstance(priorlow[0], str):
if not os.path.isfile(priorlow[0]):
mu.error("priorlow file '{:s}' not found.".format(priorlow[0]), log)
priorlow = mu.loadascii(priorlow[0])[0]
if priorup is not None and isinstance(priorup[0], str):
if not os.path.isfile(priorup[0]):
mu.error("priorup file '{:s}' not found.".format(priorup[0]), log)
priorup = mu.loadascii(priorup [0])[0]
# Process the data and uncertainties:
if data is None:
mu.error("'data' is a required argument.", log)
# If params is a filename, unpack:
elif isinstance(data[0], str):
if not os.path.isfile(data[0]):
mu.error("data file '{:s}' not found.".format(data[0]), log)
array = mu.loadbin(data[0])
data = array[0]
if len(array) == 2:
uncert = array[1]
if uncert is None:
mu.error("'uncert' is a required argument.", log)
elif isinstance(uncert[0], str):
if not os.path.isfile(uncert[0]):
mu.error("uncert file '{:s}' not found.".format(uncert[0]), log)
uncert = mu.loadbin(uncert[0])[0]
# Process the independent parameters:
if indparams != [] and isinstance(indparams[0], str):
if not os.path.isfile(indparams[0]):
mu.error("indparams file '{:s}' not found.".format(indparams[0]), log)
indparams = mu.loadbin(indparams[0])
if tracktime:
start_mpi = timeit.default_timer()
if mpi:
# Checks for mpi4py:
try:
from mpi4py import MPI
except:
mu.error("Attempted to use MPI, but mpi4py is not installed.", log)
# Get source dir:
mcfile = mc.__file__
iright = mcfile.rfind('/')
if iright == -1:
sdir = "."
else:
sdir = mcfile[:iright]
# Hack func here:
funccall = sdir + "/func.py"
if func[0] == 'hack':
funccall = func[2] + "/" + func[1] + ".py"
# Call wrapper of model function:
args = [funccall, "-c" + cfile] + remaining_argv
comm = MPI.COMM_SELF.Spawn(sys.executable, args=args, maxprocs=nprocs)
else:
comm = None
# Use a copy of uncert to avoid overwrite on it.
if uncert is not None:
unc = np.copy(uncert)
else:
unc = None
if tracktime:
start_loop = timeit.default_timer()
# Run the MCMC:
allp, bp = mc.mcmc(data, unc, func, indparams,
params, pmin, pmax, stepsize,
prior, priorlow, priorup,
numit, nchains, walk, wlike,
leastsq, chisqscale, grtest, grexit, burnin,
thinning, plots, savefile, savemodel,
comm, resume, log, rms)
if tracktime:
stop = timeit.default_timer()
# Close communications and disconnect:
if mpi:
mu.comm_disconnect(comm)
#if bench == True:
if tracktime:
mu.msg(1, "Total execution time: %10.6f s"%(stop - start), log)
if log is not None:
log.close()
from mcmc import mcmc
from ising2D import ising2D
import numpy as np
import tensorflow as tf
# -------- global constant variables ----------
L = 8
training_time = 100000
ising = ising2D(L)
seed = 71385
tf.set_random_seed(seed)
# ------ read in data from csv files -----------
trainx1 = np.loadtxt('2DIsing-MonteCarlo/bin/trainx1.dat', delimiter=',')
trainx2 = np.loadtxt('2DIsing-MonteCarlo/bin/trainx2.dat', delimiter=',')
traine1 = np.loadtxt('2DIsing-MonteCarlo/bin/traine1.dat')
traine2 = np.loadtxt('2DIsing-MonteCarlo/bin/traine2.dat')
# reshape trainy to (Ndata, 1)
traine1 = traine1.reshape((traine1.shape[0], 1))
traine2 = traine2.reshape((traine2.shape[0], 1))
mc = mcmc(L, ising)
mc.train(training_time, trainx1, trainx2, traine1, traine2)
def accuracy():
"""
Vary the confusion probability on real data.
"""
N, M, Psi_t, GT = load_dataset()
# inject confusions
conf_probs = [0.0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4]
mcmc_params = {'N_iter': 10, 'burnin': 1, 'thin': 2, 'FV': 3}
res = {'sums': [], 'mv': [], 'em': [], 'mcmc': [],
'sums_f': [], 'mv_f': [], 'em_f': [], 'mcmc_f': [],
'conf_probs': conf_probs, 'avlog': [], 'avlog_f': [],
'inv': [], 'inv_f': [], 'pinv': [], 'pinv_f': [],
'mv_p': [], 'em_p': [], 'mcmc_p': [],
'mv_f_p': [], 'em_f_p': [], 'mcmc_f_p': []}
for conf_prob in conf_probs:
mv_accu, em_accu, mcmc_accu, sums_accu, avlog_accu, inv_accu, \
pinv_accu, mv_accu_p, em_accu_p, mcmc_accu_p = [], [], [], [], [], [], [], [], [], []
mv_accu_f, em_accu_f, mcmc_accu_f, sums_accu_f, avlog_accu_f, \
inv_accu_f, pinv_accu_f, mv_accu_f_p, em_accu_f_p, mcmc_accu_f_p = [], [], [], [], [], [], [], [], [], []
accu_G_list = []
for run in range(n_runs):
GT_G, Cl, Psi = confuse(Psi_t, 1-conf_prob, GT)
# MV
mv_p = majority_voting(Psi)
mv_b = prob_binary_convert(mv_p)
# EM
em_A, em_p = expectation_maximization(N, M, Psi)
em_b = prob_binary_convert(em_p)
# MCMC
mcmc_A, mcmc_p = mcmc(N, M, Psi, mcmc_params)
mcmc_b = prob_binary_convert(mcmc_p)
data = adapter_input(Psi)
# SUMS
sums_belief = sums(N, data)
sums_b = adapter_output(sums_belief, data)
# AVG LOG
avlog_belief = average_log(N, data)
avlog_b = adapter_output(avlog_belief, data)
# INVESTMENT
inv_belief = investment(N, data)
inv_b = adapter_output(inv_belief, data)
# POOLED INVESTMENT
pinv_belief = pooled_investment(N, data)
pinv_b = adapter_output(pinv_belief, data)
# FUZZY FUSION Psi
# From now Psi is the same as Psi_fussy due to Python
f_mcmc_G, Psi_fussy = f_mcmc(N, M, deepcopy(Psi), Cl, mcmc_params)
g_accu = accu_G(f_mcmc_G, GT_G)
print g_accu
accu_G_list.append(g_accu)
data_f = adapter_input(Psi_fussy)
mv_pf = majority_voting(Psi_fussy)
mv_bf = prob_binary_convert(mv_pf)
em_Af, em_pf = expectation_maximization(N, M, Psi_fussy)
em_bf = prob_binary_convert(em_pf)
mcmc_Af, mcmc_pf = mcmc(N, M, Psi_fussy, mcmc_params)
mcmc_bf = prob_binary_convert(mcmc_pf)
sums_belief_f = sums(N, data_f)
sums_bf = adapter_output(sums_belief_f, data_f)
avlog_belief_f = average_log(N, data_f)
avlog_bf = adapter_output(avlog_belief_f, data_f)
inv_belief_f = investment(N, data_f)
inv_bf = adapter_output(inv_belief_f, data_f)
pinv_belief_f = pooled_investment(N, data_f)
pinv_bf = adapter_output(pinv_belief_f, data_f)
# exclude objects on which no conflicts
obj_with_conflicts = []
for obj_id, obj in enumerate(mv_b):
if len(obj) > 1 and obj_id in GT.keys():
obj_with_conflicts.append(obj_id)
# Binary output
mv_accu.append(np.average([mv_b[obj][GT[obj]] for obj in obj_with_conflicts]))
mv_accu_f.append(np.average([mv_bf[obj][GT[obj]] for obj in obj_with_conflicts]))
em_accu.append(np.average([em_b[obj][GT[obj]] for obj in obj_with_conflicts]))
em_accu_f.append(np.average([em_bf[obj][GT[obj]] for obj in obj_with_conflicts]))
mcmc_accu.append(np.average([mcmc_b[obj][GT[obj]] for obj in obj_with_conflicts]))
mcmc_accu_f.append(np.average([mcmc_bf[obj][GT[obj]] for obj in obj_with_conflicts]))
sums_accu.append(np.average([sums_b[obj][GT[obj]] for obj in obj_with_conflicts]))
sums_accu_f.append(np.average([sums_bf[obj][GT[obj]] for obj in obj_with_conflicts]))
avlog_accu.append(np.average([avlog_b[obj][GT[obj]] for obj in obj_with_conflicts]))
avlog_accu_f.append(np.average([avlog_bf[obj][GT[obj]] for obj in obj_with_conflicts]))
inv_accu.append(np.average([inv_b[obj][GT[obj]] for obj in obj_with_conflicts]))
inv_accu_f.append(np.average([inv_bf[obj][GT[obj]] for obj in obj_with_conflicts]))
pinv_accu.append(np.average([pinv_b[obj][GT[obj]] for obj in obj_with_conflicts]))
pinv_accu_f.append(np.average([pinv_bf[obj][GT[obj]] for obj in obj_with_conflicts]))
# Probability as output
mv_accu_p.append(np.average([mv_p[obj][GT[obj]] for obj in obj_with_conflicts]))
mv_accu_f_p.append(np.average([mv_pf[obj][GT[obj]] for obj in obj_with_conflicts]))
em_accu_p.append(np.average([em_p[obj][GT[obj]] for obj in obj_with_conflicts]))
em_accu_f_p.append(np.average([em_pf[obj][GT[obj]] for obj in obj_with_conflicts]))
mcmc_accu_p.append(np.average([mcmc_p[obj][GT[obj]] for obj in obj_with_conflicts]))
mcmc_accu_f_p.append(np.average([mcmc_pf[obj][GT[obj]] for obj in obj_with_conflicts]))
res['mv'].append(np.average(mv_accu))
res['mv_f'].append(np.average(mv_accu_f))
res['em'].append(np.average(em_accu))
res['em_f'].append(np.average(em_accu_f))
res['mcmc'].append(np.average(mcmc_accu))
res['mcmc_f'].append(np.average(mcmc_accu_f))
res['sums'].append(np.average(sums_accu))
res['sums_f'].append(np.average(sums_accu_f))
res['avlog'].append(np.average(avlog_accu))
res['avlog_f'].append(np.average(avlog_accu_f))
res['inv'].append(np.average(inv_accu))
res['inv_f'].append(np.average(inv_accu_f))
res['pinv'].append(np.average(pinv_accu))
res['pinv_f'].append(np.average(pinv_accu_f))
res['mv_p'].append(np.average(mv_accu_p))
res['mv_f_p'].append(np.average(mv_accu_f_p))
res['em_p'].append(np.average(em_accu_p))
res['em_f_p'].append(np.average(em_accu_f_p))
res['mcmc_p'].append(np.average(mcmc_accu_p))
res['mcmc_f_p'].append(np.average(mcmc_accu_f_p))
print('G Accu: {:1.4f}+-{:1.4f}'.format(np.average(accu_G_list), np.std(accu_G_list)))
print 'BINARY:'
print('confusion probability: {}, mv: {:1.4f}, em: {:1.4f}, mcmc: {:1.4f}, '
'sums: {:1.4f}, avlog: {:1.4f}, inv: {:1.4f}, pinv: {:1.4f}'
.format(conf_prob,
np.average(mv_accu),
np.average(em_accu),
np.average(mcmc_accu),
np.average(sums_accu),
np.average(avlog_accu),
np.average(inv_accu),
np.average(pinv_accu)
))
print('confusion probability: {}, mv_f: {:1.4f}, em:_f {:1.4f}, mcmc_f: {:1.4f}, '
'sums_f: {:1.4f}, avlog_f: {:1.4f}, inv_f: {:1.4f}, pinv_f: {:1.4f}'
.format(conf_prob,
np.average(mv_accu_f),
np.average(em_accu_f),
np.average(mcmc_accu_f),
np.average(sums_accu_f),
np.average(avlog_accu_f),
np.average(inv_accu_f),
np.average(pinv_accu_f)
))
print 'PROBABILISTIC:'
print('confusion probability: {}, mv_p: {:1.4f}, em_p: {:1.4f}, mcmc_p: {:1.4f}'
.format(conf_prob,
np.average(mv_accu_p),
np.average(em_accu_p),
np.average(mcmc_accu_p)
))
print('confusion probability: {}, mv_f_p: {:1.4f}, em:_f_p {:1.4f}, mcmc_f_p: {:1.4f}'
.format(conf_prob,
np.average(mv_accu_f_p),
np.average(em_accu_f_p),
np.average(mcmc_accu_f_p)
))
print 'BINARY:'
print('confusion probability: {}, mv: {:1.4f}, em: {:1.4f}, mcmc: {:1.4f}, '
'sums: {:1.4f}, avlog: {:1.4f}, inv: {:1.4f}, pinv: {:1.4f}'
.format(conf_prob,
np.std(mv_accu),
np.std(em_accu),
np.std(mcmc_accu),
np.std(sums_accu),
np.std(avlog_accu),
np.std(inv_accu),
np.std(pinv_accu)
))
print('confusion probability: {}, mv_f: {:1.4f}, em:_f {:1.4f}, mcmc_f: {:1.4f}, '
'sums_f: {:1.4f}, avlog_f: {:1.4f}, inv_f: {:1.4f}, pinv_f: {:1.4f}'
.format(conf_prob,
np.std(mv_accu_f),
np.std(em_accu_f),
np.std(mcmc_accu_f),
np.std(sums_accu_f),
np.std(avlog_accu_f),
np.std(inv_accu_f),
np.std(pinv_accu_f)
))
print 'PROBABILISTIC:'
print('confusion probability: {}, mv_p: {:1.4f}, em_p: {:1.4f}, mcmc_p: {:1.4f}'
.format(conf_prob,
np.std(mv_accu_p),
np.std(em_accu_p),
np.std(mcmc_accu_p)
))
print('confusion probability: {}, mv_f_p: {:1.4f}, em:_f_p {:1.4f}, mcmc_f_p: {:1.4f}'
.format(conf_prob,
np.std(mv_accu_f_p),
np.std(em_accu_f_p),
np.std(mcmc_accu_f_p)
))
pd.DataFrame(res).to_csv('flight_accuracy.csv', index=False)
meta_prefit['mortality, 0-1, stunted'] = P['mortality']['0-1']['stunted'] meta_prefit['mortality, 1-6, non-stunted'] = P['mortality']['1-6']['non-stunted'] meta_prefit['mortality, 1-6, stunted'] = P['mortality']['1-6']['stunted'] meta_prefit['mortality, 6-12, non-stunted'] = P['mortality']['6-12']['non-stunted'] meta_prefit['mortality, 6-12, stunted'] = P['mortality']['6-12']['stunted'] meta_prefit['mortality, 12-24, non-stunted'] = P['mortality']['12-24']['non-stunted'] meta_prefit['mortality, 12-24, stunted'] = P['mortality']['12-24']['stunted'] meta_prefit['mortality, 24-72, non-stunted'] = P['mortality']['24-72']['non-stunted'] meta_prefit['mortality, 24-72, stunted'] = P['mortality']['24-72']['stunted'] nwalkers=20 nsteps=50 nburn=10 storage="mcmc" mc = mcmc(meta_prefit, C0, P, D, t_steps, pops, nwalkers=20, nsteps=50, nburn=10, storage="mcmc") P['mortality']['0-1']['non-stunted'] = np.mean(mc[:,0]) P['mortality']['0-1']['stunted'] = np.mean(mc[:,1]) P['mortality']['1-6']['non-stunted'] = np.mean(mc[:,2]) P['mortality']['1-6']['stunted'] = np.mean(mc[:,3]) P['mortality']['6-12']['non-stunted'] = np.mean(mc[:,4]) P['mortality']['6-12']['stunted'] = np.mean(mc[:,5]) P['mortality']['12-24']['non-stunted'] = np.mean(mc[:,6]) P['mortality']['12-24']['stunted'] = np.mean(mc[:,7]) P['mortality']['24-72']['non-stunted'] = np.mean(mc[:,8]) P['mortality']['24-72']['stunted'] = np.mean(mc[:,9]) y = steps(P, C0, t_steps) M = model_outcomes(C0, P, D, t_steps, pops)
def main():
"""
Multi-Core Markov-Chain Monte Carlo (MC cubed)
This code calls MCMC to work under an MPI multiprocessor protocol or
single-thread mode. When using MPI it will launch one CPU per MCMC chain
to work in parallel.
Parameters:
-----------
cfile: String
Filename of a configuration file.
Modification History:
---------------------
2014-04-19 patricio Initial implementation. [email protected]
2014-05-04 patricio Added cfile argument for Interpreter support.
2014-05-26 patricio Re-engineered the MPI support.
2014-06-26 patricio Fixed bug with copy when uncert is None.
2014-09-14 patricio Write/read now binary files.
2014-10-23 patricio Added support for func hack.
2015-02-04 patricio Added resume argument.
2015-05-15 patricio Added logfile argument.
"""
# Parse the config file from the command line:
cparser = argparse.ArgumentParser(
description=__doc__,
add_help=False,
formatter_class=argparse.RawDescriptionHelpFormatter)
# Add config file option:
cparser.add_argument("-c",
"--config_file",
help="Configuration file",
metavar="FILE")
# Remaining_argv contains all other command-line-arguments:
args, remaining_argv = cparser.parse_known_args()
# Take configuration file from command-line:
cfile = args.config_file
# Incorrect configuration file name:
if cfile is not None and not os.path.isfile(cfile):
mu.error("Configuration file: '{:s}' not found.".format(cfile))
if cfile:
config = ConfigParser.SafeConfigParser()
config.read([cfile])
defaults = dict(config.items("MCMC"))
else:
defaults = {}
# Parser for the MCMC arguments:
parser = argparse.ArgumentParser(parents=[cparser])
# MCMC Options:
group = parser.add_argument_group("MCMC General Options")
group.add_argument("-n",
"--numit",
dest="numit",
help="Number of MCMC samples [default: %(default)s]",
type=eval,
action="store",
default=100)
group.add_argument("-x",
"--nchains",
dest="nchains",
help="Number of chains [default: %(default)s]",
type=int,
action="store",
default=10)
group.add_argument("-w",
"--walk",
dest="walk",
help="Random walk algorithm [default: %(default)s]",
type=str,
action="store",
default="demc",
choices=('demc', 'mrw'))
group.add_argument("--wlikelihood",
dest="wlike",
help="Calculate the likelihood in a wavelet base "
"[default: %(default)s]",
type=eval,
action="store",
default=False)
group.add_argument("--leastsq",
dest="leastsq",
help="Perform a least-square minimization before the "
"MCMC run [default: %(default)s]",
type=eval,
action="store",
default=False)
group.add_argument(
"--chisq_scale",
dest="chisqscale",
help="Scale the data uncertainties such that the reduced "
"chi-squared = 1. [default: %(default)s]",
type=eval,
action="store",
default=False)
group.add_argument("-g",
"--gelman_rubin",
dest="grtest",
help="Run Gelman-Rubin test [default: %(default)s]",
type=eval,
action="store",
default=False)
group.add_argument("-b",
"--burnin",
help="Number of burn-in iterations (per chain) "
"[default: %(default)s]",
dest="burnin",
type=eval,
action="store",
default=0)
group.add_argument(
"-t",
"--thinning",
dest="thinning",
help="Chains thinning factor (use every thinning-th "
"iteration) for GR test and plots [default: %(default)s]",
type=int,
action="store",
default=1)
group.add_argument(
"--plots",
dest="plots",
help="If True plot parameter traces, pairwise posteriors, "
"and marginal posterior histograms [default: %(default)s]",
type=eval,
action="store",
default=False)
group.add_argument("-o",
"--save_file",
dest="savefile",
help="Output filename to store the parameter posterior "
"distributions [default: %(default)s]",
type=str,
action="store",
default="output.npy")
group.add_argument("--savemodel",
dest="savemodel",
help="Output filename to store the evaluated models "
"[default: %(default)s]",
type=str,
action="store",
default=None)
group.add_argument("--mpi",
dest="mpi",
help="Run under MPI multiprocessing [default: "
"%(default)s]",
type=eval,
action="store",
default=False)
group.add_argument("--resume",
dest="resume",
help="If True, resume a previous run (load output) "
"[default: %(default)s]",
type=eval,
action="store",
default=False)
group.add_argument("--rms",
dest="rms",
help="If True, calculate the RMS of (data-bestmodel) "
"[default: %(default)s]",
type=eval,
action="store",
default=False)
group.add_argument("--logfile",
dest="logfile",
help="Log file.",
action="store",
default=None)
group.add_argument("-T",
"--tracktime",
dest="tractime",
action="store_true")
# Fitting-parameter Options:
group = parser.add_argument_group("Fitting-function Options")
group.add_argument("-f",
"--func",
dest="func",
help="List of strings with the function name, module "
"name, and path-to-module [required]",
type=mu.parray,
action="store",
default=None)
group.add_argument("-p",
"--params",
dest="params",
help="Filename or list of initial-guess model-fitting "
"parameter [required]",
type=mu.parray,
action="store",
default=None)
group.add_argument("-m",
"--pmin",
dest="pmin",
help="Filename or list of parameter lower boundaries "
"[default: -inf]",
type=mu.parray,
action="store",
default=None)
group.add_argument("-M",
"--pmax",
dest="pmax",
help="Filename or list of parameter upper boundaries "
"[default: +inf]",
type=mu.parray,
action="store",
default=None)
group.add_argument("-s",
"--stepsize",
dest="stepsize",
help="Filename or list with proposal jump scale "
"[default: 0.1*params]",
type=mu.parray,
action="store",
default=None)
group.add_argument("-i",
"--indparams",
dest="indparams",
help="Filename or list with independent parameters for "
"func [default: None]",
type=mu.parray,
action="store",
default=[])
# Data Options:
group = parser.add_argument_group("Data Options")
group.add_argument("-d",
"--data",
dest="data",
help="Filename or list of the data being fitted "
"[required]",
type=mu.parray,
action="store",
default=None)
group.add_argument("-u",
"--uncertainties",
dest="uncert",
help="Filemane or list with the data uncertainties "
"[default: ones]",
type=mu.parray,
action="store",
default=None)
group.add_argument("--prior",
dest="prior",
help="Filename or list with parameter prior estimates "
"[default: %(default)s]",
type=mu.parray,
action="store",
default=None)
group.add_argument("--priorlow",
dest="priorlow",
help="Filename or list with prior lower uncertainties "
"[default: %(default)s]",
type=mu.parray,
action="store",
default=None)
group.add_argument("--priorup",
dest="priorup",
help="Filename or list with prior upper uncertainties "
"[default: %(default)s]",
type=mu.parray,
action="store",
default=None)
# Set the defaults from the configuration file:
parser.set_defaults(**defaults)
# Set values from command line:
args2, unknown = parser.parse_known_args(remaining_argv)
# Unpack configuration-file/command-line arguments:
numit = args2.numit
nchains = args2.nchains
walk = args2.walk
wlike = args2.wlike
leastsq = args2.leastsq
chisqscale = args2.chisqscale
grtest = args2.grtest
burnin = args2.burnin
thinning = args2.thinning
plots = args2.plots
savefile = args2.savefile
savemodel = args2.savemodel
mpi = args2.mpi
resume = args2.resume
tracktime = args2.tractime
logfile = args2.logfile
rms = args2.rms
func = args2.func
params = args2.params
pmin = args2.pmin
pmax = args2.pmax
stepsize = args2.stepsize
indparams = args2.indparams
data = args2.data
uncert = args2.uncert
prior = args2.prior
priorup = args2.priorup
priorlow = args2.priorlow
nprocs = nchains
# Open a log FILE if requested:
if logfile is not None:
log = open(logfile, "w")
else:
log = None
# Handle arguments:
if params is None:
mu.error("'params' is a required argument.", log)
elif isinstance(params[0], str):
# If params is a filename, unpack:
if not os.path.isfile(params[0]):
mu.error("'params' file not found.", log)
array = mu.read2array(params[0])
# Array size:
ninfo, ndata = np.shape(array)
if ninfo == 7: # The priors
prior = array[4]
priorlow = array[5]
priorup = array[6]
if ninfo >= 4: # The stepsize
stepsize = array[3]
if ninfo >= 2: # The boundaries
pmin = array[1]
pmax = array[2]
params = array[0] # The initial guess
# Check for pmin and pmax files if not read before:
if pmin is not None and isinstance(pmin[0], str):
if not os.path.isfile(pmin[0]):
mu.error("'pmin' file not found.", log)
pmin = mu.read2array(pmin[0])[0]
if pmax is not None and isinstance(pmax[0], str):
if not os.path.isfile(pmax[0]):
mu.error("'pmax' file not found.", log)
pmax = mu.read2array(pmax[0])[0]
# Stepsize:
if stepsize is not None and isinstance(stepsize[0], str):
if not os.path.isfile(stepsize[0]):
mu.error("'stepsize' file not found.", log)
stepsize = mu.read2array(stepsize[0])[0]
# Priors:
if prior is not None and isinstance(prior[0], str):
if not os.path.isfile(prior[0]):
mu.error("'prior' file not found.", log)
prior = mu.read2array(prior[0])[0]
if priorlow is not None and isinstance(priorlow[0], str):
if not os.path.isfile(priorlow[0]):
mu.error("'priorlow' file not found.", log)
priorlow = mu.read2array(priorlow[0])[0]
if priorup is not None and isinstance(priorup[0], str):
if not os.path.isfile(priorup[0]):
mu.error("'priorup' file not found.", log)
priorup = mu.read2array(priorup[0])[0]
# Process the data and uncertainties:
if data is None:
mu.error("'data' is a required argument.", log)
# If params is a filename, unpack:
elif isinstance(data[0], str):
if not os.path.isfile(data[0]):
mu.error("'data' file not found.", log)
array = mu.readbin(data[0])
data = array[0]
if len(array) == 2:
uncert = array[1]
if uncert is not None and isinstance(uncert[0], str):
if not os.path.isfile(uncert[0]):
mu.error("'uncert' file not found.", log)
uncert = mu.readbin(uncert[0])[0]
# Process the independent parameters:
if indparams != [] and isinstance(indparams[0], str):
if not os.path.isfile(indparams[0]):
mu.error("'indparams' file not found.", log)
indparams = mu.readbin(indparams[0])
if tracktime:
start_mpi = timeit.default_timer()
if mpi:
# Checks for mpi4py:
try:
from mpi4py import MPI
except:
mu.error("Attempted to use MPI, but mpi4py is not installed.", log)
# Get source dir:
mcfile = mc.__file__
iright = mcfile.rfind('/')
if iright == -1:
sdir = "."
else:
sdir = mcfile[:iright]
# Hack func here:
funccall = sdir + "/func.py"
if func[0] == 'hack':
funccall = func[2] + "/" + func[1] + ".py"
# Call wrapper of model function:
args = [funccall, "-c" + cfile] + remaining_argv
comm = MPI.COMM_SELF.Spawn(sys.executable, args=args, maxprocs=nprocs)
else:
comm = None
# Use a copy of uncert to avoid overwrite on it.
if uncert is not None:
unc = np.copy(uncert)
else:
unc = None
if tracktime:
start_loop = timeit.default_timer()
# Run the MCMC:
allp, bp = mc.mcmc(data, unc, func, indparams, params, pmin, pmax,
stepsize, prior, priorlow, priorup, numit, nchains,
walk, wlike, leastsq, chisqscale, grtest, burnin,
thinning, plots, savefile, savemodel, comm, resume, log,
rms)
if tracktime:
stop = timeit.default_timer()
# Close communications and disconnect:
if mpi:
mu.comm_disconnect(comm)
#if bench == True:
if tracktime:
mu.msg(1, "Total execution time: %10.6f s" % (stop - start), log)
if log is not None:
log.close()
### loop mcmc
import time
print 'start', time.asctime()
n_step = int(5e5)
n_para = 3
para_chain = np.zeros((n_step, n_planet*n_para))
loglikelihood_chain = np.zeros((n_step, n_planet))
p0 = np.array([0.3, 0.3, 2. ]) # alpha, beta, mass
p_stepsize = np.array([1e-2, 1e-2, 0.2])
for i in range(n_planet):
para_chain[:, (i*n_para):((i+1)*n_para)], loglikelihood_chain[:, i] = mcmc( p0, p_stepsize, n_step, loglikelihood, \
data=[mass_obs[i], mass_err[i], rad_obs[i], rad_err[i]], domain=domain )
print 'end', time.asctime()
### print
np.savetxt('test_marcy_cluster_parameter.out',para_chain)
np.savetxt('test_marcy_cluster_loglike.out',loglikelihood_chain)
### plot
row = 2
col = 4
f, ((a00,a01,a02,a03),(a10,a11,a12,a13))=plt.subplots(row,col,figsize=(col*5,row*5))
import sys
WINDOWS = 100
T = 5.0
def get_probs(samples, expression):
length = (len(expression) / WINDOWS) + 1
counts = [0] * length
for sample in samples:
for ratchet in sample:
counts[ratchet] += 1
max_expression = max(expression)
for i in xrange(len(counts)):
counts[i] = (counts[i] * max_expression) / float(len(samples))
return counts
data = open(sys.argv[1], 'r')
c = 0
with open(sys.argv[1]) as fp:
for line in fp:
start, end = line.split()[1:3]
if int(end)-int(start) < WINDOWS*7: continue
expression = [int(i) for i in line.strip().split('\t')[-1].split(",")]
c += 1
if line.split('\t')[-2] == "-":
expression.reverse()
samples = mcmc(expression, WINDOWS, T)
print line.strip() + '\t' + ','.join(map(str, get_probs(samples, expression)))
popt, pcov = curve_fit(gaussianfit, x, y, sigma=dy, p0=[0.5, 1, 3, 0])
x2 = np.linspace(-5, 8, 2500)
theta1 = np.zeros((4))
guess1 = gaussian(x, [popt[0], popt[1], popt[2], popt[3]])
plt.figure(figsize=(4.13, 5.1625))
gs = gridspec.GridSpec(3, 2)
ax1 = plt.subplot(gs[2, :])
ax1.errorbar(
x, y, marker="o", ls="", yerr=dy, c=sns.color_palette()[0]
)
import mcmc
history = mcmc.mcmc(popt, gaussian, 0.1, [x, y, dy], 600, 100)
for i in range(0, history.shape[0], 5):
ax1.plot(
x2,
gaussian(
x2,
[
history[i, 0],
history[i, 1],
history[i, 2],
history[i, 3],
],
),
alpha=0.05,
c=sns.color_palette()[2],
and we can use chi2 to check if the model is good ''' def log_likely(para, model_func, x_real, y_data, y_err): y_model = model_func(x_real,para) return 0.-sum( (y_model - y_data)**2./(2.* y_err**2.) ) def chi2(para, model_func, x_real, y_data, y_err): y_model = model_func(x_real,para) return sum( (y_model - y_data)**2./(2.* y_err**2.) ) ### test with 1st/2nd/3rd order polynomial p0 = np.zeros(2) p_step = np.ones(2) * 0.3 poly1_seq, chi1_seq = mcmc(p0, p_step, n_step, log_likely, \ [model_poly1,x_real,y_data,y_err], seed, chi2) p0 = np.zeros(3) p_step = np.ones(3) * 0.3 poly2_seq, chi2_seq = mcmc(p0, p_step, n_step, log_likely, \ [model_poly2,x_real,y_data,y_err], seed, chi2) p0 = np.zeros(4) p_step = np.ones(4) * 0.3 poly3_seq, chi3_seq = mcmc(p0, p_step, n_step, log_likely, \ [model_poly3,x_real,y_data,y_err], seed, chi2)
import pymc
import mcmc
from pymads import *
p = pesting.read_pest('exp_model.pst')
mcmc.create_pymc_model(p)
M = mcmc.mcmc(p, nsample=10)
def efficiency():
"""
Efficiency as the number of clusters growing.
"""
N, M, Psi, GT = load_dataset()
# inject confusions
# Ncs = [10, 100, 1000, 10000]
Ncs = [10000]
mcmc_params = {'N_iter': 10, 'burnin': 1, 'thin': 2, 'FV': 3}
# mcmc_params = {'N_iter': 30, 'burnin': 5, 'thin': 3, 'FV': 4}
res = {'mv': [],
'mv std': [],
'em': [],
'em std': [],
'mcmc': [],
'mcmc std': [],
'f_mcmc': [],
'f_mcmc std': [],
'sums': [],
'sums std': [],
'avlog': [],
'avlog std': [],
'inv': [],
'inv std': [],
'pinv': [],
'pinv std': [],
'number of objects with confusions': Ncs}
for Nc in Ncs:
print 'Nc: {}'.format(Nc)
times = [[], [], [], [], [], [], [], []]
for run in range(n_runs):
print 'run: {}'.format(run)
GT_G, Cl, cPsi = confuse(Psi, 0.8, GT, Nc)
start = time.time()
majority_voting(cPsi)
times[0].append(time.time() - start)
start = time.time()
expectation_maximization(N, M, cPsi)
times[1].append(time.time() - start)
start = time.time()
mcmc(N, M, cPsi, mcmc_params)
times[2].append(time.time() - start)
start = time.time()
f_mcmc(N, M, cPsi, Cl, mcmc_params)
times[3].append(time.time() - start)
data = adapter_input(cPsi)
start = time.time()
sums(N, data)
times[4].append(time.time() - start)
start = time.time()
average_log(N, data)
times[5].append(time.time() - start)
start = time.time()
investment(N, data)
times[6].append(time.time() - start)
start = time.time()
pooled_investment(N, data)
times[7].append(time.time() - start)
res['mv'].append(np.average(times[0]))
res['em'].append(np.average(times[1]))
res['mcmc'].append(np.average(times[2]))
res['f_mcmc'].append(np.average(times[3]))
res['sums'].append(np.average(times[4]))
res['avlog'].append(np.average(times[5]))
res['inv'].append(np.average(times[6]))
res['pinv'].append(np.average(times[7]))
res['mv std'].append(np.std(times[0]))
res['em std'].append(np.std(times[1]))
res['mcmc std'].append(np.std(times[2]))
res['f_mcmc std'].append(np.std(times[3]))
res['sums std'].append(np.std(times[4]))
res['avlog std'].append(np.std(times[5]))
res['inv std'].append(np.std(times[6]))
res['pinv std'].append(np.std(times[7]))
print(
'{}\tmv: {:1.4f}\tem: {:1.4f}\tmcmc: {:1.4f}\tf_mcmc: {:1.4f}\tsums: {:1.4f}\tavlog: {:1.4f}\tinv: {:1.4f}\tpinv: {:1.4f}'.format(
Nc,
np.average(times[0]),
np.average(times[1]),
np.average(times[2]),
np.average(times[3]),
np.average(times[4]),
np.average(times[5]),
np.average(times[6]),
np.average(times[7])
))
print('{}\tmv: {:1.4f}\tem: {:1.4f}\tmcmc: {:1.4f}\tf_mcmc: {:1.4f}\tsums: {:1.4f}\tavlog: {:1.4f}\tinv: {:1.4f}\tpinv: {:1.4f}'.format(Nc,
np.average(times[0]),
np.average(times[1]),
np.average(times[2]),
np.average(times[3]),
np.average(times[4]),
np.average(times[5]),
np.average(times[6]),
np.average(times[7])
))
pd.DataFrame(res).to_csv(work_dir + 'experiments_results/flight_efficiency.csv', index=False)
plt.figure(figsize=(12,8))
plt.plot(log_r_center, log_rho, label='$\mathrm{data}$')
plt.plot(log_r_center, fit_0, label='$\mathrm{first\;guess}$')
plt.xlabel(r'$\log{(r)}$')
plt.ylabel(r'$\log{(\rho (r))}$')
plt.legend(loc='lower left')
plt.savefig('./mcmc_plots/density_profile_firstguess.png')
#fit
log_rho_0, log_r_c, alpha, beta, log_rho_0_std, log_r_c_std, alpha_std, beta_std = mcmc(log_r_center, log_rho)
fit = model(log_r_center, log_rho_0, log_r_c, alpha, beta)
#plot fit
plt.figure(figsize=(12,8))
plt.plot(log_r_center, log_rho, label='$\mathrm{data}$')
plt.plot(log_r_center, fit, label='$\mathrm{fit}$')
plt.xlabel(r'$\log{(r)}$')
plt.ylabel(r'$\log{(\rho (r))}$')
plt.legend(loc='lower left')
