def checkPar(self, par, num, section_name='plot'):
for jj in range(num):
# try:
# if self._data[par[jj]].min() == self._data[par[jj]].max():
# af.WarningNoWait("Parameter %s=%f is a cosntant number. No plot for it. "%(par[jj], self._data[par[jj]].min()))
# return False
# except KeyError:
# af.WarningNoWait("Parameter '%s' in [plot] section do not exist. No plot for it."%( par[jj] ) )
# return False
if par[jj] not in self._data.columns:
af.WarningNoWait(
"Parameter '%s' in [%s] section do not exist." %
(par[jj], section_name))
return False
if self._data.shape[0] == 0:
af.WarningNoWait("Parameter '%s' in [%s] section is empty." %
(par[jj], section_name))
return False
if not numpy.issubdtype(self._data[par[jj]].dtype, numpy.number):
af.WarningNoWait(
"Parameter %s in [%s] section is not float number." %
(par[jj], section_name))
return False
return True
def getPlot(self, ScanMethod):
try:
import matplotlib
except ImportError:
af.ErrorStop("No matplotlib module. No plot will be generated.")
matplotlib.use('Agg')
import matplotlib.pyplot as plt
if len(self._Histogram) + len(self._Scatter) + len(self._Color) + len(
self._Contour) == 0:
return
af.Info('Start to plot the result ... ')
FigNames = []
for ii in self._Histogram:
histconf = {
'bins': 50,
'normed': False,
'facecolor': 'green',
'alpha': 0.7
}
af.Info('Generate histogram plot of parameter %s' % ii[0])
if not self.checkPar(ii, 1): continue
f = plt.figure(**figconf)
subplot = f.add_subplot(111)
subplot.hist(self._data[ii[0]], **histconf)
subplot.set_xlabel(ii[0], **labelconf)
subplot.set_ylabel('Count', **labelconf)
subplot.tick_params(which='both', direction='out')
plt.savefig(os.path.join(self._path, ii[1]))
for ii in self._Scatter:
scatterconf = {
's': 50,
'marker': 'o',
'edgecolors': 'None',
'alpha': 0.9
}
af.Info('Generate scatter plot of parameter %s and %s' %
(ii[0], ii[1]))
if not self.checkPar(ii, 2): continue
f = plt.figure(**figconf)
subplot = f.add_subplot(111)
subplot.scatter(self._data[ii[0]], self._data[ii[1]],
**scatterconf)
subplot.set_xlabel(ii[0], **labelconf)
subplot.set_ylabel(ii[1], **labelconf)
subplot.tick_params(which='both', direction='out')
plt.savefig(os.path.join(self._path, ii[2]))
if ScanMethod == af._mcmc:
f = plt.figure(**figconf)
subplot = f.add_subplot(111)
subplot.scatter(self._dataAllTry[ii[0]],
self._dataAllTry[ii[1]],
label='All',
**scatterconf)
subplot.scatter(self._data[ii[0]],
self._data[ii[1]],
label='Surviving',
**scatterconf)
plt.legend(**legendconf)
subplot.set_xlabel(ii[0], **labelconf)
subplot.set_ylabel(ii[1], **labelconf)
subplot.tick_params(which='both', direction='out')
plt.savefig(os.path.join(self._path, 'Compare_%s' % ii[2]))
for ii in self._Color:
colorconf = {'edgecolors': 'None', 'cmap': plt.get_cmap('winter')}
af.Info(
'Generate color plot of parameter %s and %s with color %s' %
(ii[0], ii[1], ii[2]))
if not self.checkPar(ii, 3): continue
f = plt.figure(**figconf)
subplot = f.add_subplot(111)
weigh = 50
if "probability" in self._data.columns:
weigh = self._data["probability"] * 100 / self._data[
"probability"].max() + 20
elif "mult" in self._data.columns:
weigh = self._data["mult"] * 100 / self._data["mult"].max() + 20
sc1 = subplot.scatter(self._data[ii[0]],
self._data[ii[1]],
c=self._data[ii[2]],
s=weigh,
**colorconf)
cb1 = plt.colorbar(sc1)
cb1.set_label(ii[2], **labelconf)
subplot.set_xlabel(ii[0], **labelconf)
subplot.set_ylabel(ii[1], **labelconf)
subplot.tick_params(which='both', direction='out')
plt.savefig(os.path.join(self._path, ii[3]))
for ii in self._Contour:
try:
from scipy.interpolate import griddata
except ImportError:
af.WarningNoWait(
"No scipy module. Contour plot will not be generated.")
break
af.Info(
'Generate contour plot of parameter %s and %s with contour %s'
% (ii[0], ii[1], ii[2]))
if not self.checkPar(ii, 3): continue
f = plt.figure(**figconf)
subplot = f.add_subplot(111)
x = self._data[ii[0]]
X = numpy.linspace(min(x), max(x), 100)
y = self._data[ii[1]]
Y = numpy.linspace(min(y), max(y), 100)
# z = [ numpy.log10(abs(u)) for u in self._data[ii[2]] ] # log10
z = self._data[ii[2]]
Z = griddata(x, y, z, X, Y, interp='linear')
#C = subplot.contour(X,Y,Z, 3,linewidths=2)
#plt.clabel(C, inline=True, fontsize=8)
## debug
#Cpoint = self.get_contour_verts(C)
#numpy.savetxt(os.path.join(self._path, "contour_1_1.dat"),Cpoint[0][0])
C = subplot.contourf(X, Y, Z, 3, cmap=plt.cm.rainbow)
cb1 = plt.colorbar(C)
cb1.set_label(ii[2], **labelconf)
subplot.set_xlabel(ii[0], **labelconf)
subplot.set_ylabel(ii[1], **labelconf)
subplot.tick_params(which='both', direction='out')
plt.savefig(os.path.join(self._path, ii[3]))
def mcmcrun(LnLike, Prior, n_params, n_live_points, inpar, fixedpar, outpar,
StepSize, AccepRate, FlagTuneR, InitVal, n_print, outputfolder):
data_file = open(os.path.join(outputfolder, af.ResultFile), 'a')
all_data_file = open(os.path.join(outputfolder, af.ResultFile_MCMC), 'a')
file_path = os.path.join(outputfolder, "SavedFile")
n_dims = len(inpar)
# Initialise cube
cube = [af.NaN] * n_params
covar = [] # the sigma of gauss distribution, normalized to 1
par = [] # test par, normalized to 1
CurPar = [] # current par, normalized to 1
for i, name in enumerate(inpar):
covar.append(StepSize[name])
cube[i] = InitVal[name]
par.append(cube[i])
CurPar.append(cube[i])
n_init = 0
while True:
Prior(cube, n_dims, n_params) # normalized to cube to real value
lnlike = LnLike(cube, n_dims, n_params)
AllOutMCMC = cube.copy()
AllOutMCMC.append(1)
#"True" for saving files of initial physical point
saveCube(AllOutMCMC, all_data_file, file_path, '0', False)
if lnlike > af.log_zero / 2.0: break
if n_init == 0:
af.WarningNoWait(
'The initial point is unphysical, it will find the physical initial points randmly.'
)
n_init = n_init + 1
if n_init > 100:
af.ErrorStop(
'Can not find physical initial points with 100 tries.')
for i in range(n_dims):
cube[i] = random()
CurPar[i] = cube[i]
CurObs = []
CurChisq = -2.0 * lnlike
for i in range(n_params):
CurObs.append(cube[i])
CurObs.append(0) # mult
printPoint(0, cube, n_dims, inpar, fixedpar, outpar, lnlike, 0)
# Initialize the MCMC parameters
MinChisq = CurChisq
Chisq = CurChisq
Nrun = 0
Naccept = 0
Nout = 0
mult = 1
kcovar = 0
while Naccept < n_live_points:
#Nrun += 1
RangeFlag = True
for j in range(n_dims):
par[j] = gauss(CurPar[j],
exp(kcovar) * covar[j]) # normalized to 1
#rd = random()
#par[j] = CurPar[j] + covar[j] * (0.5-rd)*2
if max(par) > 1 or min(par) < 0:
RangeFlag = False
Nout = Nout + 1
if Nout % 100 == 0:
af.WarningNoWait("Too many points out of range!")
else:
Nrun += 1
Nout = 0
for i in range(n_dims):
cube[i] = par[i]
Prior(cube, n_dims, n_params)
lnlike = LnLike(cube, n_dims, n_params)
AllOutMCMC = cube.copy()
AllOutMCMC.append(1)
saveCube(AllOutMCMC, all_data_file, file_path, '0', False)
Chisq = -2.0 * lnlike
Flag_accept = RangeFlag and (Chisq < CurChisq + 20)
if Flag_accept:
if CurChisq > Chisq:
Flag_accept = True
else:
Flag_accept = random() < exp(CurChisq - Chisq)
if Flag_accept:
CurObs[-1] = mult
#"Naccept+1" due to file of Chisq have covered file of CurChisq
saveCube(CurObs, data_file, file_path, str(Naccept + 1), True)
CurChisq = Chisq
for i in range(n_params):
CurObs[i] = cube[i]
for i in range(n_dims):
CurPar[i] = par[i]
if Chisq < MinChisq: MinChisq = Chisq
Naccept += 1
mult = 1
else:
if RangeFlag:
mult += 1
AccRat = float(Naccept) / float(Nrun)
if FlagTuneR and Nrun < 1000:
kcovar = kcovar + 1.0 / (float(Nrun)**0.7) * (AccRat - AccepRate)
else:
kcovar = 1
if Nrun % n_print == 0:
if RangeFlag:
printPoint(Nrun, cube, n_dims, inpar, fixedpar, outpar, lnlike,
Naccept)
printPoint4MCMC(Chisq, CurChisq, MinChisq, AccRat, FlagTuneR,
kcovar)
# save the last point
CurObs[-1] = mult
saveCube(CurObs, data_file, file_path, str(Naccept), True)
def setInputPar(self, inputvar):
inputvar = af.string2nestlist(inputvar)
# inputvar is list of list of input parameters define in section [scan]
af.Info('Input parameters = ')
for ii in inputvar:
lenii = len(ii)
if self._ScanMethod == af._postprocess:
self.InPar[ii[0]] = af.NaN
self.AllPar[ii[0]] = af.NaN
af.Info(' ID= %s, read from previous '%(ii[0]))
continue
if lenii < 3 :
af.ErrorStop(self.InputCheck(ii[0], 3, "Value"))
# Set fixed par
if ii[1].upper() == "FIXED":
if lenii > 3 :
af.WarningNoWait(self.InputCheck(ii[0], 3, "Value"))
af.WarningWait("The rest %i values will be ignore."%(lenii-3) )
af.Info(' ID= %s\tPrior= %s\t =%f'%(ii[0],ii[1],ii[2]))
self.FixedPar[ii[0]] = ii[2]
self.AllPar[ii[0]] = ii[2]
continue
# Initialize other input par to NaN
self.InputPar[ii[0]] = ii
self.InPar[ii[0]] = af.NaN
self.AllPar[ii[0]] = af.NaN
if lenii < 4 :
af.ErrorStop(self.InputCheck(ii[0], 4, "Minimum, Maximum"))
if self._ScanMethod in [af._random, af._multinest]:
if lenii > 4 :
af.WarningNoWait(self.InputCheck(ii[0], 4, "Minimum, Maximum"))
af.WarningWait("The rest %i values will be ignore."%(lenii-4) )
af.Info(' ID= %s\tPrior= %s\tMin= %f\tMax= %f'%(ii[0],ii[1],ii[2],ii[3]))
continue
if self._ScanMethod == af._grid:
if lenii == 4:
self.GridBin[ii[0]]=20
af.WarningNoWait(self.InputCheck(ii[0], 5, "Minimum, Maximum, Number of bins"))
af.WarningWait("'Number of bins' will take default value, 20.")
else:
self.GridBin[ii[0]]=ii[4]
if self.GridBin[ii[0]] < 0 or type(ii[4]) != int:
af.WarningNoWait(InputCheck(ii[0], 5, "Minimum, Maximum, Number of bins"))
af.ErrorStop("'Number of bins' is not a positive integer.")
if lenii> 5:
af.WarningNoWait(self.InputCheck(ii[0], 5, "Minimum, Maximum, Number of bins"))
af.WarningWait("The rest %i values will be ignore."%(lenii-5) )
af.Info(' ID= %s\tPrior= %s\tMin= %f\tMax= %f\tNbin=%i'%(ii[0],ii[1],ii[2],ii[3],self.GridBin[ii[0]]))
continue
if self._ScanMethod == af._mcmc:
if lenii < 6:
af.WarningNoWait(self.InputCheck(ii[0], 6, "Minimum, Maximum, Interval, Initial value"))
self.MCMCiv[ii[0]] = 1./2.
IniV = float(ii[3]+ii[2])/2.
af.WarningWait("'Initial value' will take default value, (Max-Min)/2.")
if lenii < 5:
self.MCMCss[ii[0]] = 1./30.
Step = float(ii[3]-ii[2])/30.
af.WarningWait("'Interval' will take default value, (Max-Min)/30.")
else:
# The scan range is normalized to 1
self.MCMCss[ii[0]] = 1.0/float(ii[4])
Step = float(ii[3]-ii[2])/float(ii[4])
if ii[1].lower() == 'flat':
self.MCMCiv[ii[0]] = float(ii[5]-ii[2])/float(ii[3]-ii[2])
elif ii[1].lower() == 'log':
self.MCMCiv[ii[0]] = (log10(ii[5])-log10(ii[2]))/(log10(ii[3]) - log10(ii[2]))
IniV = ii[5]
if lenii > 6:
af.WarningNoWait(self.InputCheck(ii[0], 6, "Minimum, Maximum, Interval, Initial value"))
af.WarningWait("The rest %i values will be ignore."%(lenii-6) )
af.Info(' ID= %s\tPrior= %s\tMin= %f\tMax= %f\tStep=%f\tIniV=%f'%(ii[0],ii[1],ii[2],ii[3],Step,self.MCMCiv[ii[0]]))
continue
def ReadIn(Configfile, ES, Programs, Constraint, Ploter):
cf = configparser.ConfigParser()
cf.read(Configfile)
# Read the basic scan parameters
if not ('scan' in cf.sections()):
af.ErrorStop(notFind('[scan] section'))
try:
ES.setScanMethod(cf.get('scan', 'Scan method'))
except configparser.NoOptionError:
af.ErrorStop(notFind('Scan method'))
try:
ES.setFolderName(cf.get('scan', 'Result folder name'))
except configparser.NoOptionError:
af.ErrorStop(notFind('Result folder name'))
# Read the plot information
# This part is put here, because if ScanMethod = PLOT, the rest of sections can be ignore.
plot_items = []
try:
plot_items = cf.options("plot")
except configparser.NoSectionError:
if ES.getScanMethod() == af._plot:
af.ErrorStop(notFind('[plot] section'))
if 'histogram' in plot_items:
Ploter.setHistogram(cf.get('plot', 'Histogram'))
if 'scatter' in plot_items:
Ploter.setScatter(cf.get('plot', 'Scatter'))
if 'color' in plot_items:
Ploter.setColor(cf.get('plot', 'Color'))
if 'contour' in plot_items:
Ploter.setContour(cf.get('plot', 'Contour'))
checkDuplicatedName(Ploter._FigNames, "Figure", False)
if ES.getScanMethod() == af._plot: return []
# Back to read the basic scan parameters
try:
ES.setPointNum(cf.getint('scan', 'Number of points'))
if ES.getScanMethod() in af._no_random:
af.WarningNoWait(
'"Number of points" in configure file is not used.')
except configparser.NoOptionError:
if ES.getScanMethod() in af._no_random:
pass
else:
af.WarningWait(notFind('Number of points') + takeDefault('2'))
except ValueError:
af.ErrorStop(notInteger("Number of points"))
try:
ES.setRandomSeed(cf.getint('scan', 'Random seed'))
except configparser.NoOptionError:
if ES.getScanMethod() not in af._no_random:
af.Info("Use current system time as random seed.")
except ValueError:
af.ErrorStop(notInteger("Random seed"))
try:
ES.setPrintNum(cf.getint('scan', 'Interval of print'))
except configparser.NoOptionError:
af.Info(notFind('Interval of print') + takeDefault('1'))
ES.setPrintNum('1')
except ValueError:
af.WarningNoWait(notInteger("Interval of print") + takeDefault('1'))
ES.setPrintNum('1')
try:
ES.setInputPar(cf.get('scan', 'Input parameters'))
except configparser.NoOptionError:
af.ErrorStop(notFind('Input parameters'))
checkDuplicatedName(list(ES.InPar.keys()), "Input parameter")
try:
ES.setAccepRate(cf.get('scan', 'Acceptance rate'))
except configparser.NoOptionError:
if ES.getScanMethod() == af._mcmc:
af.Info(notFind('Acceptance rate') + takeDefault('0.25'))
else:
pass
## sort programs by ID
ProgID = [x for x in cf.sections() if x.startswith('program')]
if len(ProgID) == 0:
af.ErrorStop(notFind('[program] section'))
for ii in ProgID:
if not str.isdigit(ii[7:]):
af.ErrorStop('The section name of [%s] is wrong' % ii)
ProgID = sorted(ProgID, key=lambda x: int(x[7:]))
## Read the programs sections
outputVarNames = []
for ii in ProgID:
items = cf.options(ii)
Programs[ii] = PROGRAM()
if 'program name' in items:
Programs[ii].setProgName(cf.get(ii, 'Program name'))
else:
Programs[ii].setProgName(ii)
if 'execute command' in items:
Programs[ii].setCommand(cf.get(ii, 'Execute command'))
else:
af.ErrorStop('No "Execute command" in "%s".' % ii)
if 'command path' in items:
Programs[ii].setComPath(cf.get(ii, 'Command path'))
else:
Programs[ii].setComPath('')
af.WarningNoWait('Use current path as "Command path" for "%s".' %
ii)
if 'input file' in items and 'input variable' in items:
Programs[ii].setInputFile(cf.get(ii, 'Input file'))
Programs[ii].setInputVar(cf.get(ii, 'Input variable'))
if 'output file' in items and 'output variable' in items:
Programs[ii].setOutputFile(cf.get(ii, 'Output file'))
Programs[ii].setOutputVar(cf.get(ii, 'Output variable'))
for key, item in Programs[ii]._OutputVar.items():
for subitem in item:
outputVarNames.append(subitem[0])
checkDuplicatedName(outputVarNames, "Output variable")
# Additional optional commands
try:
Programs[ii].setExecutor(cf.get(ii, 'Command executor'))
except:
af.Info('Use "os.system" execute commands.')
try:
Programs[ii].setOutputClean(cf.get(ii, 'Clean output file'))
except:
af.Info('Delete output file(s) of %s before execute it. ' % ii)
try:
Programs[ii].setBound(cf.get(ii, 'Bound'))
except:
af.Info('No Bound.')
if Programs[ii]._executor: # 'os.system'
try:
Programs[ii].setTimeLimit(
cf.getfloat(ii, 'Time limit in minute'))
af.WarningNoWait('Time limit of %s is not used.' % ii)
except:
pass
else: # 'subprocess.popen'
try:
Programs[ii].setTimeLimit(
cf.getfloat(ii, 'Time limit in minute'))
except:
af.Info('No time limit setting. Using defaut value 60 min.')
## Read the constraints
constraint_items = []
try:
constraint_items = cf.options("constraint")
if len(constraint_items) == 0 and (ES.getScanMethod()
not in af._no_like):
af.ErrorStop(
'Section [constraint] is empty in the configure file.')
except configparser.NoSectionError:
if ES.getScanMethod() in af._no_like:
pass
else:
af.ErrorStop(notFind('[constraint] section'))
af.Info('...............................................')
af.Debug('Constraint items:', constraint_items)
if 'gaussian' in constraint_items:
Constraint.setGaussian(cf.get('constraint', 'Gaussian'))
# In order to add "math .." in "Gaussian" to self.AllPar TODO
if Programs:
Programs[ProgID[0]].setGaussian(cf.get('constraint', 'Gaussian'))
if 'freeformchi2' in constraint_items:
Constraint.setFreeFormChi2(cf.get('constraint', 'FreeFormChi2'))
if Programs:
Programs[ProgID[0]].setFreeFormChi2(
cf.get('constraint', 'FreeFormChi2'))
if ('gaussian' not in constraint_items) and (
'freeformchi2' not in constraint_items) and (ES.getScanMethod()
not in af._no_like):
af.ErrorStop('No valid iterm in [constraint] section.')
ES.setProgram(Programs)
return ProgID