def _extract_top(rootname, extract_limit, extract_llkmin, extract_step, verbose):
"""private"""
runfile = "%s/_HerrMet.run" % rootname
with RunFile(runfile, verbose=verbose) as rundb:
print "extract : llkmin %f, limit %d, step %d" % (extract_llkmin, extract_limit, extract_step),
modelids, chainids, weights, llks, nlayers, dms, srs = zip(*list(
rundb.getpack(limit=extract_limit,
llkmin=extract_llkmin,
step=extract_step,
algo="METROPOLIS")))
if len(modelids) > 100:
raise ValueError('too many models to extract')
for rank, (modelid, chainid, weight, llk, nlayer, dm, sr) in \
enumerate(zip(modelids, chainids, weights, llks, nlayers, dms, srs)):
try:
out = '{}/_HerrMet.rank{}.modelid{}.chainid{}.llk{}.mod96'.format(
rootname, rank, modelid, chainid, llk)
if verbose:
print "writing %s" % out
dm.write96(out) # , overwrite=True)
except KeyboardInterrupt:
raise
except Exception as e:
print "Error", str(e)
def gen(rootnames, runmode):
for rootname in rootnames:
targetfile = "%s/_HerrMet.target" % rootname
paramfile = "%s/_HerrMet.param" % rootname
runfile = "%s/_HerrMet.run" % rootname
if runmode == "append" and not os.path.exists(runfile):
runmode = "restart"
elif runmode == "restart" and os.path.exists(runfile):
os.remove(runfile)
elif runmode == "skip" and os.path.exists(runfile):
print "skip %s" % rootname
continue
# ------
p, logRHOM = load_paramfile(paramfile)
# ------
d = makedatacoder(targetfile, which=Datacoder_log) # datacoder based on observations
dobs, CDinv = d.target()
duncs = CDinv ** -.5
ND = len(dobs)
dinfs = d(0.1 * np.ones_like(d.values))
dsups = d(3.5 * np.ones_like(d.values))
logRHOD = LogGaussND(dobs, duncs, dinfs, dsups, k=1000., nanbehavior=1)
# ------
G = Theory(parameterizer=p, datacoder=d)
# ---------------------------------
if runmode == "restart" or runmode == "skip":
with RunFile(runfile, create=True, verbose=verbose) as rundb:
rundb.drop()
rundb.reset(p.NLAYER, d.waves, d.types, d.modes, d.freqs)
elif runmode == "append":
pass
else:
raise Exception('unexpected runmode %s' % runmode)
# ---------------------------------
for chainid in xrange(Nchain):
M0 = np.random.rand(len(p.MINF)) * (p.MSUP - p.MINF) + p.MINF
MSTD = p.MSTD
yield Job(runfile=runfile,
rootname=rootname,
chainid=chainid,
M0=M0,
MSTD=MSTD,
G=G,
ND=ND,
logRHOD=logRHOD,
logRHOM=logRHOM,
p=p, d=d,
nkeep=Nkeep,
verbose=verbose)
def gen(rootnames):
for rootname in rootnames:
targetfile = "%s/_HerrMet.target" % rootname
paramfile = "%s/_HerrMet.param" % rootname
runfile = "%s/_HerrMet.run" % rootname
# ------
p, logRHOM = load_paramfile(paramfile)
# ------
d = makedatacoder(targetfile, which=Datacoder_log) # datacoder based on observations
dobs, CDinv = d.target()
duncs = CDinv ** -.5
ND = len(dobs)
dinfs = d(0.1 * np.ones_like(d.values))
dsups = d(3.5 * np.ones_like(d.values))
logRHOD = LogGaussND(dobs, duncs, dinfs, dsups, k=1000., nanbehavior=1)
# ------
G = Theory(parameterizer=p, datacoder=d)
# ---------------------------------
with RunFile(runfile, verbose=verbose) as rundb:
best = list(rundb.get(llkmin=top_llkmin, limit=top_limit, step=top_step, algo=None))
# ---------------------------------
for modelid, chainid, weight, llk, nlayer, model, dat in best:
M0 = p(*model)
DM = 1.0 #p.MSTD
yield Job(runfile=runfile,
rootname=rootname,
chainid=chainid,
M0=M0,
DM=DM,
G=G,
ND=ND,
logRHOD=logRHOD,
logRHOM=logRHOM,
p=p, d=d,
verbose=verbose)
def _extract_pdf(rootname, extract_mode, extract_limit, extract_llkmin, extract_step, verbose, percentiles, mapkwargs):
"""private"""
percentiles = np.array(percentiles, float)
assert len(np.unique(percentiles)) == len(percentiles)
assert np.all(0 < percentiles)
assert np.all(1 > percentiles)
assert np.all(percentiles[1:] > percentiles[:-1])
runfile = "%s/_HerrMet.run" % rootname
with RunFile(runfile, verbose=verbose) as rundb:
print "extract : llkmin %f, limit %d, step %d" % (extract_llkmin, extract_limit, extract_step),
if extract_mode == "best":
chainids, weights, llks, ms, ds = \
rundb.getzip(limit=extract_limit,
llkmin=extract_llkmin,
step=extract_step,
algo="METROPOLIS")
elif extract_mode == "last":
chainids, weights, llks, ms, ds = \
rundb.getlastszip(limit=extract_limit,
llkmin=extract_llkmin,
step=extract_step,
algo="METROPOLIS")
else:
raise Exception('unexpected extract mode %s' % extract_mode)
dms = [depthmodel_from_arrays(ztop, vp, vs, rh) for ztop, vp, vs, rh in ms]
for p, (vppc, vspc, rhpc, prpc) in \
dmstats1(dms,
percentiles=percentiles,
Ndepth=100,
Nvalue=100,
weights=weights,
**mapkwargs):
try:
dmout = depthmodel(vppc, vspc, rhpc)
out = '%s/_HerrMet.p%.2f.mod96' % (rootname, p)
if verbose:
print "writing %s" % out
dmout.write96(out) # , overwrite=True)
except KeyboardInterrupt:
raise
except Exception as e:
print "Error", str(e)
for p in percentiles:
out = '%s/_HerrMet.p%.2f.surf96' % (rootname, p)
os.system('trash %s' % out)
for p, (wpc, tpc, mpc, fpc, vpc) in \
dispstats(ds,
percentiles=percentiles,
Ndisp=100,
weights=weights,
**mapkwargs):
try:
srout = surf96reader_from_arrays(wpc, tpc, mpc, fpc, vpc)
out = '%s/_HerrMet.p%.2f.surf96' % (rootname, p)
if verbose:
print "writing to %s" % out
with open(out, 'a') as fid:
fid.write(srout.__str__())
fid.write('\n')
except KeyboardInterrupt:
raise
except Exception as e:
print "Error", str(e)
def _extract_pdf(rootname, extract_mode, extract_limit, extract_llkmin,
extract_step, verbose, percentiles, mapkwargs):
"""private"""
percentiles = np.array(percentiles, float)
assert len(np.unique(percentiles)) == len(percentiles)
assert np.all(0 < percentiles)
assert np.all(1 > percentiles)
assert np.all(percentiles[1:] > percentiles[:-1])
for p in percentiles:
extract_disp_file = EXTRACTDISPFILE.format(
rootname=rootname,
extract_mode=extract_mode,
extract_limit=extract_limit,
extract_llkmin=extract_llkmin,
extract_step=extract_step,
percentile=p)
extract_model_file = EXTRACTMODELFILE.format(
rootname=rootname,
extract_mode=extract_mode,
extract_limit=extract_limit,
extract_llkmin=extract_llkmin,
extract_step=extract_step,
percentile=p)
if os.path.isfile(extract_disp_file) and os.path.isfile(
extract_model_file):
continue
break
else:
# did not break => means all files exist already
print('found existing extraction files in {}, skip'.format(rootname))
return
try:
runfile = _find_runfile(rootname)
except IOError:
return
with RunFile(runfile, verbose=verbose) as rundb:
print "extract : llkmin %f, limit %d, step %d" % (
extract_llkmin, extract_limit, extract_step),
if extract_mode == "best":
chainids, weights, llks, ms, ds = \
rundb.getzip(limit=extract_limit,
llkmin=extract_llkmin,
step=extract_step,
algo="METROPOLIS")
elif extract_mode == "last":
chainids, weights, llks, ms, ds = \
rundb.getlastszip(limit=extract_limit,
llkmin=extract_llkmin,
step=extract_step,
algo="METROPOLIS")
else:
raise Exception('unexpected extract mode %s' % extract_mode)
if not len(ms):
return
dms = [depthmodel_from_arrays(ztop, vp, vs, rh) for ztop, vp, vs, rh in ms]
for p, (vppc, vspc, rhpc, prpc) in \
dmstats1(dms,
percentiles=percentiles,
Ndepth=100,
Nvalue=100,
weights=weights,
**mapkwargs):
try:
dmout = depthmodel(vppc, vspc, rhpc)
extract_model_file = EXTRACTMODELFILE.format(
rootname=rootname,
extract_mode=extract_mode,
extract_limit=extract_limit,
extract_llkmin=extract_llkmin,
extract_step=extract_step,
percentile=p)
if verbose:
print "writing %s" % extract_model_file
dmout.write96(extract_model_file) # , overwrite=True)
except KeyboardInterrupt:
raise
except Exception as e:
print "Error", str(e)
for p in percentiles:
extract_disp_file = EXTRACTDISPFILE.format(
rootname=rootname,
extract_mode=extract_mode,
extract_limit=extract_limit,
extract_llkmin=extract_llkmin,
extract_step=extract_step,
percentile=p)
if os.path.isfile(extract_disp_file):
os.system('trash {}'.format(extract_disp_file))
for p, (wpc, tpc, mpc, fpc, vpc) in \
dispstats(ds,
percentiles=percentiles,
Ndisp=100,
weights=weights,
**mapkwargs):
try:
srout = surf96reader_from_arrays(wpc, tpc, mpc, fpc, vpc)
extract_disp_file = EXTRACTDISPFILE.format(
rootname=rootname,
extract_mode=extract_mode,
extract_limit=extract_limit,
extract_llkmin=extract_llkmin,
extract_step=extract_step,
percentile=p)
if verbose:
print "writing to {}".format(extract_disp_file)
with open(extract_disp_file, 'a') as fid:
fid.write(srout.__str__())
fid.write('\n')
except KeyboardInterrupt:
raise
except Exception as e:
print "Error", str(e)
def _display_function(rootname, argv, verbose, mapkwargs):
"""private"""
targetfile = "%s/_HerrMet.target" % rootname
paramfile = "%s/_HerrMet.param" % rootname
runfile = '%s/_HerrMet.run' % rootname
pngfile = '%s/_HerrMet.png' % rootname
#HerrLininitfile = '%s/_HerrLin.init' % rootname
# ------ Initiate the displayer using the target data if exists
if "-compact" in argv.keys(): # compact mode
which_displayer = DepthDispDisplayCompact
else:
which_displayer = DepthDispDisplay
if os.path.exists(targetfile):
rd = which_displayer(targetfile=targetfile)
d = makedatacoder(
targetfile, which=Datacoder_log) # datacoder based on observations
dobs, _ = d.target()
else:
print "no target file found in %s" % rootname
rd = which_displayer()
# ------ Display run results if exist
if os.path.exists(runfile) and ("-plot" in argv.keys()
or "-pdf" in argv.keys()):
with RunFile(runfile, verbose=verbose) as rundb:
s = rundb.select('select MODELID from MODELS limit 1')
if s is not None:
# --- display best models
if "-plot" in argv.keys():
assert argv["-plot"] == [] or len(
argv["-plot"]) == 4 # unexpected argument number
if argv["-plot"] == []:
plot_mode, plot_limit, plot_llkmin, plot_step = \
default_plot_mode, default_plot_limit, \
default_plot_llkmin, default_plot_step
elif len(argv['-plot']) == 4:
plot_mode, plot_limit, plot_llkmin, plot_step = argv[
'-plot']
else:
raise Exception()
print "plot : %s, limit %d, llkmin %f, step %d" % (
plot_mode, plot_limit, plot_llkmin, plot_step),
if plot_mode == "best":
chainids, weights, llks, ms, ds = \
rundb.getzip(limit=plot_limit,
llkmin=plot_llkmin,
step=plot_step,
algo="METROPOLIS")
elif plot_mode == "last":
chainids, weights, llks, ms, ds = \
rundb.getlastszip(limit=plot_limit,
llkmin=plot_llkmin,
step=plot_step,
algo="METROPOLIS")
else:
raise Exception('unexpected plot mode %s' % plot_mode)
vmin, vmax = llks.min(), llks.max()
# colors = values2colors(llks, vmin=vmin, vmax=vmax, cmap=argv['-cmap'])
if "-overdisp" in argv.keys():
"""note : recomputing dispersion with another frequency array might
result in a completely different dispersion curve in case
of root search failure """
waves, types, modes, freqs, _ = ds[0]
overwaves, overtypes, overmodes, _, _ = zip(*list(
groupbywtm(waves, types, modes, freqs,
np.arange(len(freqs)), None, True)))
overfreqs = [
freqspace(0.6 * min(freqs), 1.4 * max(freqs), 100,
"plog") for _ in xrange(len(overwaves))
]
overwaves, overtypes, overmodes, overfreqs = \
igroupbywtm(overwaves, overtypes, overmodes, overfreqs)
for llk, (mms, dds) in zip(
llks[::-1],
overdisp(ms[::-1],
overwaves,
overtypes,
overmodes,
overfreqs,
verbose=verbose,
**mapkwargs)):
# rd.plotmodel(color=clr, alpha=1.0, linewidth=3, *mms)
rd.addmodel(colorvalue=llk, *mms)
try:
# rd.plotdisp(color=clr, alpha=1.0, linewidth=3, *dds)
rd.adddisp(colorvalue=llk, *dds)
except KeyboardInterrupt:
raise
except Exception as e:
print "Error : could not plot dispersion curve (%s)" % str(
e)
# cb = makecolorbar(vmin=vmin, vmax=vmax, cmap=argv['-cmap'])
# pos = rd.axdisp[-1].get_position()
# cax = rd.fig.add_axes((pos.x0, 0.12, pos.width, 0.01))
# rd.fig.colorbar(cb, cax=cax, label="log likelyhood", orientation="horizontal")
else:
"display the dispersion curves as stored in the database"
for i in range(len(llks))[::-1]:
# rd.plotmodel(color=colors[i], alpha=1.0, linewidth=3, *ms[i])
# rd.plotdisp(color=colors[i], alpha=1.0, linewidth=3, *ds[i])
rd.addmodel(colorvalue=llks[i], *ms[i])
rd.adddisp(colorvalue=llks[i], *ds[i])
# cb = makecolorbar(vmin=vmin, vmax=vmax, cmap=argv['-cmap'])
# pos = rd.axdisp[-1].get_position()
# cax = rd.fig.add_axes((pos.x0, 0.12, pos.width, 0.01))
# rd.fig.colorbar(cb, cax=cax, label="log likelyhood", orientation="horizontal")
# cax.set_xticklabels(cax.get_xticklabels(), rotation=90., horizontalalignment="center")
rd.showdispcoll(vmin=vmin,
vmax=vmax,
cmap=argv['-cmap'],
alpha=1.0,
linewidth=3)
rd.showdepthcoll(vmin=vmin,
vmax=vmax,
cmap=argv['-cmap'],
alpha=1.0,
linewidth=3)
rd.colorbar(vmin=vmin,
vmax=vmax,
cmap=argv['-cmap'],
label="log likelyhood",
orientation="horizontal")
print rd.cax.get_position()
rd.cax.set_xticklabels(rd.cax.get_xticklabels(),
rotation=90.,
horizontalalignment="center")
# ---- display posterior pdf
if "-pdf" in argv.keys():
assert argv["-pdf"] == [] or len(
argv["-pdf"]) == 4 # unexpected argument number
if argv["-pdf"] == []:
pdf_mode, pdf_limit, pdf_llkmin, pdf_step = \
default_pdf_mode, default_pdf_limit, default_pdf_llkmin, default_pdf_step
elif len(argv['-pdf']) == 4:
pdf_mode, pdf_limit, pdf_llkmin, pdf_step = argv[
'-pdf']
else:
raise Exception()
print "pdf : %s, limit %d, llkmin %f, step %d" % (
pdf_mode, pdf_limit, pdf_llkmin, pdf_step),
if pdf_mode == "best":
chainids, weights, llks, ms, ds = \
rundb.getzip(limit=pdf_limit,
llkmin=pdf_llkmin,
step=pdf_step,
algo="METROPOLIS")
elif pdf_mode == "last":
chainids, weights, llks, ms, ds = \
rundb.getlastszip(limit=pdf_limit,
llkmin=pdf_llkmin,
step=pdf_step,
algo="METROPOLIS")
else:
raise Exception('unexpected pdf mode %s' % pdf_mode)
dms = [
depthmodel_from_arrays(ztop, vp, vs, rh)
for ztop, vp, vs, rh in ms
]
# display percentiles of model and data pdfs
clr = "b" if "-plot" not in argv.keys() else "k"
alp = 1.0 if "-plot" not in argv.keys() else 0.5
for p, (vppc, vspc, rhpc, prpc) in \
dmstats1(dms,
percentiles=[0.01, 0.16, 0.5, 0.84, 0.99],
Ndepth=100,
Nvalue=100,
weights=weights,
**mapkwargs):
try:
l = 3 if p == 0.5 else 1
for what, where in zip([vppc, vspc, rhpc, prpc], [
rd.axdepth['VP'], rd.axdepth['VS'],
rd.axdepth['RH'], rd.axdepth['PR']
]):
if where is not None:
what.show(where,
color=clr,
linewidth=l,
alpha=alp)
except KeyboardInterrupt:
raise
except Exception as e:
print "Error", str(e)
# display the disp pdf
for p, (wpc, tpc, mpc, fpc, vpc) in \
dispstats(ds,
percentiles=[0.01, 0.16, 0.5, 0.84, 0.99],
Ndisp=100,
weights=weights,
**mapkwargs):
try:
l = 3 if p == 0.5 else 1
rd.plotdisp(wpc,
tpc,
mpc,
fpc,
vpc,
dvalues=None,
color=clr,
alpha=alp,
linewidth=l)
except KeyboardInterrupt:
raise
except Exception as e:
print "Error", str(e)
# ------
if os.path.exists(paramfile):
p, _ = load_paramfile(paramfile)
showvp, showvs, showrh, showpr = True, True, True, True
if isinstance(p, Parameterizer_mZVSVPRH):
showpr = False
elif isinstance(p, Parameterizer_mZVSPRRH):
showvp = False
elif isinstance(p, Parameterizer_mZVSPRzRHvp):
showvp = showpr = showrh = False
elif isinstance(p, Parameterizer_mZVSPRzRHz):
showvp = showpr = showrh = False
else:
raise Exception('')
#
vplow, vphgh, vslow, vshgh, rhlow, rhhgh, prlow, prhgh = p.boundaries()
for what, where in zip(\
[vplow, vphgh, vslow, vshgh, rhlow, rhhgh, prlow, prhgh],
[rd.axdepth['VP'], rd.axdepth['VP'], rd.axdepth['VS'], rd.axdepth['VS'], rd.axdepth['RH'], rd.axdepth['RH'], rd.axdepth['PR'], rd.axdepth['PR']]):
if where is not None:
what.show(where,
alpha=1.0,
color="k",
marker="o--",
linewidth=1,
markersize=3)
zmax = 1.1 * p.inv(p.MINF)[0][-1]
if isinstance(p, Parameterizer_mZVSPRzRHvp):
rd.axdepth['PR'].plot(p.PRz(np.linspace(0., zmax, 100)),
np.linspace(0., zmax, 100),
"r--",
linewidth=3)
legendtext(rd.axdepth['PR'], p.PRzName, loc=4)
legendtext(rd.axdepth['RH'], p.RHvpName, loc=4)
elif isinstance(p, Parameterizer_mZVSPRzRHz):
rd.axdepth['PR'].plot(p.PRz(np.linspace(0., zmax, 100)),
np.linspace(0., zmax, 100),
"r--",
linewidth=3)
rd.axdepth['RH'].plot(p.RHz(np.linspace(0., zmax, 100)),
np.linspace(0., zmax, 100),
"r--",
linewidth=3)
legendtext(rd.axdepth['PR'], p.PRzName, loc=4)
legendtext(rd.axdepth['RH'], p.RHzName, loc=4)
rd.set_zlim(np.array([0, zmax]))
else:
print "call option --param to see prior depth boundaries"
# --------------------
if "-m96" in argv.keys(): # plot user data on top
for m96 in argv['-m96']:
try:
dm = depthmodel_from_mod96(m96)
dm.vp.show(rd.axdepth['VP'], "m", linewidth=3, label=m96)
dm.vs.show(rd.axdepth['VS'], "m", linewidth=3)
dm.rh.show(rd.axdepth['RH'], "m", linewidth=3)
dm.pr().show(rd.axdepth['PR'], "m", linewidth=3)
except KeyboardInterrupt:
raise
except: #Exception as e:
print 'could not read or display %s (reason : %s)' % (m96,
str(e))
rd.axdepth['VP'].legend(loc=3)
if "-ritt" in argv.keys():
a = AsciiFile('/mnt/labex2/home/max/data/boreholes/GRT1/GRT1.logsonic')
for what, where in zip(
[a.data['VS'], a.data['VP'], a.data['VP'] / a.data['VS']],
[rd.axdepth['VS'], rd.axdepth['VP'], rd.axdepth['PR']]):
if where is not None:
where.plot(what, a.data['TVD'] / 1000., "m", alpha=0.5)
# --------------------
if os.path.exists(targetfile):
# plot data on top
rd.plotdisp(d.waves,
d.types,
d.modes,
d.freqs,
d.inv(dobs),
dvalues=d.dvalues,
alpha=.5,
color="r",
linewidth=2)
if "-overdisp" in argv.keys():
rd.set_vlim((0.5 * d.values.min(), 1.5 * d.values.max()))
rd.set_plim((0.8 / overfreqs.max(), 1.2 / overfreqs.min()))
else:
rd.set_vlim((0.8 * d.values.min(), 1.2 * d.values.max()))
rd.set_plim((0.8 / d.freqs.max(), 1.2 / d.freqs.min()))
rd.tick()
rd.grid()
rd.fig.suptitle(rootname.split('_HerrMet_')[-1])
if "-ftsz" in argv.keys():
chftsz(rd.fig, argv["-ftsz"][0])
else:
chftsz(rd.fig, default_fontsize)
if "-png" in argv.keys():
dpi = argv['-png'][0] if len(argv['-png']) else default_dpi
if verbose:
print "writing %s" % pngfile
rd.fig.savefig(pngfile, dpi=dpi)
elif "-inline" in argv.keys():
plt.show()
else:
showme()
plt.close(rd.fig)
def run(argv, verbose, mapkwargs):
for k in argv.keys():
if k in ['main', "_keyorder"]:
continue # private keys
if k not in authorized_keys:
raise Exception('option %s is not recognized' % k)
rootnames = argv['main']
if rootnames == []:
rootnames = glob.glob(default_rootnames)
assert len(rootnames)
runmode = argv['-mode'][0] if "-mode" in argv.keys() else default_mode
assert runmode in ['restart', 'append', 'skip']
Nchain = int(
argv['-nchain'][0]) if "-nchain" in argv.keys() else default_nchain
Nkeep = int(
argv['-nkeep'][0]) if "-nkeep" in argv.keys() else default_nkeep
# ------------------------
def gen(rootnames, runmode):
for rootname in rootnames:
targetfile = "%s/_HerrMet.target" % rootname
paramfile = "%s/_HerrMet.param" % rootname
runfile = "%s/_HerrMet.run" % rootname
if runmode == "append" and not os.path.exists(runfile):
runmode = "restart"
elif runmode == "restart" and os.path.exists(runfile):
os.remove(runfile)
elif runmode == "skip" and os.path.exists(runfile):
print("skip %s" % rootname)
continue
print(rootname)
# ------
p, logRHOM = load_paramfile(paramfile)
# ------
d = makedatacoder(
targetfile,
which=Datacoder_log) # datacoder based on observations
dobs, CDinv = d.target()
duncs = CDinv**-.5
ND = len(dobs)
dinfs = d(0.1 * np.ones_like(d.values))
dsups = d(3.8 * np.ones_like(d.values))
logRHOD = LogGaussND(dobs,
duncs,
dinfs,
dsups,
k=1000.,
nanbehavior=1)
# ------
G = Theory(parameterizer=p, datacoder=d)
# ---------------------------------
if runmode == "restart" or runmode == "skip":
with RunFile(runfile, create=True, verbose=verbose) as rundb:
rundb.drop()
rundb.reset(p.NLAYER, d.waves, d.types, d.modes, d.freqs)
elif runmode == "append":
pass
else:
raise Exception('unexpected runmode %s' % runmode)
# ---------------------------------
for chainid in range(Nchain):
M0 = np.random.rand(len(p.MINF)) * (p.MSUP - p.MINF) + p.MINF
MSTD = p.MSTD
yield Job(runfile=runfile,
rootname=rootname,
chainid=chainid,
M0=M0,
MSTD=MSTD,
G=G,
ND=ND,
logRHOD=logRHOD,
logRHOM=logRHOM,
p=p,
d=d,
nkeep=Nkeep,
verbose=verbose)
# ---------------------------------
def fun(worker, rootname, runfile, chainid, M0, MSTD, G, ND, logRHOD,
logRHOM, p, d, nkeep, verbose):
models, datas, weights, llks = metropolis(
M0,
MSTD,
G,
ND,
logRHOD,
logRHOM,
nkeep=nkeep,
normallaw=worker.randn,
unilaw=worker.rand,
chainid=chainid,
HL=10,
IK0=0.25,
MPMIN=1.e-6,
MPMAX=1e6,
adjustspeed=0.3,
nofail=True,
debug=False,
verbose=verbose,
head="%10s " % rootname.split('_HerrMet_')[-1])
I = np.any(~np.isnan(datas), axis=1)
models, datas, weights, llks = models[I, :], datas[
I, :], weights[I], llks[I]
return runfile, models, datas, weights, llks, p, d
# ---------------------------------
with MapAsync(fun, gen(rootnames, runmode), **mapkwargs) as ma:
for jobid, answer, _, _ in ma:
runfile, models, datas, weights, llks, p, d = answer
if verbose:
print('=> write to %s' % runfile)
with RunFile(runfile, verbose=False) as rundb:
rundb.begintransaction()
try:
rundb.insert("METROPOLIS", models, datas, weights, llks, p,
d)
rundb.commit()
except:
rundb.rollback(crash=True)
def manage(argv, verbose, mapkwargs):
for k in argv.keys():
if k in ['main', "_keyorder"]:
continue # private keys
if k not in authorized_keys:
raise Exception('option %s is not recognized' % k)
rootnames0 = argv['main']
if rootnames0 == []:
rootnames0 = glob.glob(default_rootnames)
assert len(rootnames0)
# exclude rootnames with no run file
rootnames = []
runfiles = []
for rootname in rootnames0:
runfile = "%s/_HerrMet.run" % rootname
if os.path.exists(runfile):
rootnames.append(rootname)
runfiles.append(runfile)
else:
pass #print "%s : %s does not exist" % (rootname, runfile)
del rootnames0
assert len(rootnames) and len(rootnames) == len(runfiles)
# summarize all files
for rootname, runfile in zip(rootnames, runfiles):
with RunFile(runfile, verbose=False) as rundb:
rundb.summary(head=rootname + " : ")
showfun = showme
if "-inline" in argv.keys():
showfun = plt.show
# more options
if np.any([
opt in argv.keys()
for opt in "-stats", "-delbad", "-delchains", "-plot"
]):
for rootname, runfile in zip(rootnames, runfiles):
with RunFile(runfile, verbose=verbose) as rundb:
# ------------ print chains stats
if "-stats" in argv.keys():
rundb.stats(head=rootname + " : ")
# ------------ rm
if "-delbad" in argv.keys():
rundb.del_bad(llkmin=argv['-delbad'][0])
if "-delchains" in argv.keys():
rundb.del_chain(chainid=argv['-delchains'])
# ------------ plot
if "-plot" in argv.keys():
vmin = argv['-plot'][0] if len(argv['-plot']) else None
s = rundb.select('''
select CHAINID, group_concat(NITER), group_concat(LLK) from MODELS
group by CHAINID
''')
plt.figure(figsize=(8, 4))
ax0 = gcf().add_subplot(121)
ax1 = gcf().add_subplot(122, sharey=ax0)
LLKs = []
for CHAINID, NITER, LLK in s:
NITER = np.asarray(NITER.split(','), int)
LLK = np.asarray(LLK.split(','), float)
ax0.plot(NITER, LLK)
ax0.text(NITER[-1], LLK[-1], CHAINID)
LLKs = np.concatenate((LLKs, LLK))
ax1.plot(np.sort(LLKs)[::-1])
if vmin is not None:
ax0.set_ylim(vmin, 0)
ax0.set_xlabel('# iteration')
ax0.set_ylabel('log likelihood')
ax1.set_xlabel('# rank')
ax0.grid(True, linestyle=":")
ax1.grid(True, linestyle=":")
gcf().suptitle(rootname.split('_HerrMet_')[-1])
showfun()
gcf().savefig("%s/_HerrMet.stats.png" % rootname)
plt.close(gcf())
def neldermead(argv, verbose, mapkwargs):
raise Exception('not ready')
for k in argv.keys():
if k in ['main', "_keyorder"]:
continue # private keys
if k not in authorized_keys:
raise Exception('option %s is not recognized' % k)
rootnames = argv['main']
if rootnames == []:
rootnames = glob.glob(default_rootnames)
assert len(rootnames)
for rootname in rootnames:
runfile = "%s/_HerrMet.run" % rootname
assert os.path.exists(runfile)
assert argv["-top"] == [] or len(argv["-top"]) == 3 # unexpected argument number
if argv["-top"] == []:
top_llkmin, top_limit, top_step = default_top_llkmin, default_top_limit, default_top_step
elif len(argv['-top']) == 3:
top_llkmin, top_limit, top_step = argv['-top']
print "top : llkmin %f, limit %d, step %d" % (top_llkmin, top_limit, top_step)
# ------------------------
def gen(rootnames):
for rootname in rootnames:
targetfile = "%s/_HerrMet.target" % rootname
paramfile = "%s/_HerrMet.param" % rootname
runfile = "%s/_HerrMet.run" % rootname
# ------
p, logRHOM = load_paramfile(paramfile)
# ------
d = makedatacoder(targetfile, which=Datacoder_log) # datacoder based on observations
dobs, CDinv = d.target()
duncs = CDinv ** -.5
ND = len(dobs)
dinfs = d(0.1 * np.ones_like(d.values))
dsups = d(3.5 * np.ones_like(d.values))
logRHOD = LogGaussND(dobs, duncs, dinfs, dsups, k=1000., nanbehavior=1)
# ------
G = Theory(parameterizer=p, datacoder=d)
# ---------------------------------
with RunFile(runfile, verbose=verbose) as rundb:
best = list(rundb.get(llkmin=top_llkmin, limit=top_limit, step=top_step, algo=None))
# ---------------------------------
for modelid, chainid, weight, llk, nlayer, model, dat in best:
M0 = p(*model)
DM = 1.0 #p.MSTD
yield Job(runfile=runfile,
rootname=rootname,
chainid=chainid,
M0=M0,
DM=DM,
G=G,
ND=ND,
logRHOD=logRHOD,
logRHOM=logRHOM,
p=p, d=d,
verbose=verbose)
def fun(runfile, rootname, chainid, M0, DM, G, ND, logRHOD, logRHOM, p, d, verbose):
models, datas, llks = neldermead_function(M0, DM, G, ND, logRHOD, logRHOM,
alpha=1.0,
beta=0.9,
gamma=1.2,
maxiter=1000,
interrupt=1e-12,
debug=1)
weights = np.ones_like(llks)
I = np.any(~np.isnan(datas), axis=1)
models, datas, weights, llks = models[I, :], datas[I, :], weights[I], llks[I]
I = argunique(llks)
models, datas, weights, llks = models[I, :], datas[I, :], weights[I], llks[I]
return runfile, models, datas, weights, llks, p, d
with MapAsync(fun, gen(rootnames), **mapkwargs) as ma:
for jobid, answer, _, _ in ma:
runfile, models, datas, weights, llks, p, d = answer
if verbose:
print '=> write to %s' % runfile
with RunFile(runfile, verbose=False) as rundb:
rundb.begintransaction()
try:
rundb.insert("NELDERMEAD", models, datas, weights, llks, p, d)
rundb.commit()
except:
rundb.rollback(crash=True)
rd = DepthDispDisplay(targetfile=runfile.replace('.run', '.target'))
for model, data, llk in zip(models, datas, llks):
rd.plotmodel(color="k", alpha=0.2, showvp=True, showvs=True, showrh=True,
showpr=True, *p.inv(model))
rd.plotdisp(d.waves,d.types,d.modes,d.freqs,d.inv(data), dvalues=None, color="k", alpha=0.2)
showme()
plt.close('all')