def EMG2DFit(x, y, data, x01h, samplewd, nSample, **kwargs):
#data is 2-D distribution of columns (e.g. AOD, SO2 mol/m2)
# initial guess of each parameter
#x,y always in the grid resolution (dxy==1)
#7 parameters to fit, similar in two different parameterizations
#samplewd: resolution, i.e. how many km represented by 1 grid
npar = 7 #(a,x0,ux,uy,sigma,eta,b)
GauDecay = False
if 'GauDecay' in kwargs:
if kwargs['GauDecay'] == True:
GauDecay = True
if 'fixparam' in kwargs:
fixpars = kwargs['fixparam']
fixvals = kwargs['fixvalue']
# if GauDecay == False:
# eta=kwargs['eta']
if 'solver' in kwargs:
solver = kwargs['solver']
else:
solver = 'trust-constr'
#total emission (kg)
ini_a = np.sum(data) #dxy==1
#location of maxima
maxxW = x[np.nanargmax(data)]
maxyW = y[np.nanargmax(data)]
if np.array(maxxW).size > 1:
maxxW = maxxW[0]
maxyW = maxyW[0]
#take the across-wind data at maxima to estimate sigma
ydata = data[x == maxxW]
peaky = y[x == maxxW]
HM = (np.max(ydata) - np.min(ydata)) / 2. + np.min(ydata)
FWHM = np.absolute(peaky[np.nanargmin(np.absolute(ydata - HM))] -
maxyW) * 2. / 2.355
minchisquare = -1
#latin hypercube construction of initials
design = lhs(3, samples=nSample)
if GauDecay == False:
means = np.array([ini_a, x01h, FWHM])
else:
means = np.array([ini_a / x01h, x01h, FWHM])
stds = nSample * means
for i in np.arange(3):
design[:, i] = norm(loc=means[i], scale=stds[i]).ppf(design[:, i])
for jfit in np.arange(nSample + 1):
if jfit < nSample:
fitpars = design[jfit, :]
else:
fitpars = means
if np.min(fitpars) < 0:
continue
try:
if solver == 'trust-constr': # Do the scipy.optimize fit using "trust_constr" method
# initials and bounds
x0 = np.zeros(npar) # (a,x0,ux,uy,sigma,eta,b)
x0[0:2] = fitpars[0:2]
x0[2] = maxxW
x0[3] = maxyW
x0[4] = fitpars[2]
x0[5] = np.min(data)
x0[6] = 1.5 * samplewd
bounds = Bounds([0., 1. / 3, np.min(x), np.min(y), 0.5, 0.,0.], \
[np.inf, np.inf, np.max(x), np.max(y), np.inf, np.max(data),np.inf])
if 'fixparam' in kwargs:
for ipar in np.arange(len(fixvals)):
bounds.lb[fixpars[ipar]] = fixvals[ipar]
bounds.ub[fixpars[ipar]] = fixvals[ipar]
x0[fixpars[ipar]] = fixvals[ipar]
if GauDecay == False:
# EMG Fitting
nlconstr = NonlinearConstraint(EMG2DConstr, -np.inf, 20.)
res = scipyminimize(EMG2Dchisqr, x0, args=(x, y, data), method='trust-constr', \
constraints=[nlconstr], options={'verbose': 0,'maxiter':10000}, \
bounds=bounds) # constraints=[lconstr, nlconstr],
pars = res.x
Hess = nd.Hessian(EMG2Dchisqr)(pars, x, y, data)
chisqr = EMG2Dchisqr(pars, x, y, data)
elif GauDecay == True:
# 2-D gaussian decay fitting
res = scipyminimize(GauDecay2Dchisqr, x0, args=(x, y, data), method='trust-constr', \
options={'verbose': 0,'maxiter':10000}, bounds=bounds) # constraints=[lconstr, nlconstr],
pars = res.x
Hess = nd.Hessian(GauDecay2Dchisqr)(pars, x, y, data)
chisqr = GauDecay2Dchisqr(pars, x, y, data)
else:
print(
"Something went wrong, the type of fitting is not determined!"
)
return [False, False]
sigmas = np.sqrt(
np.linalg.inv(Hess) * chisqr / (data.size - pars.size))
params = Parameters()
params.add('x0', value=pars[1], brute_step=sigmas[1, 1])
params.add('ux', value=pars[2], brute_step=sigmas[2, 2])
params.add('uy', value=pars[3], brute_step=sigmas[3, 3])
params.add('sigma', value=pars[4], brute_step=sigmas[4, 4])
if GauDecay == True:
params.add('Qu', value=pars[0], brute_step=sigmas[
0, 0]) # x-direction integration of total burden
else:
params.add('a', value=pars[0], brute_step=sigmas[
0, 0]) # x-direction integration of total burden
params.add('eta', value=pars[6], brute_step=sigmas[
6, 6]) # x-direction integration of total burden
params.add('b', value=pars[5], brute_step=sigmas[5, 5])
if (minchisquare < 0) | (chisqr < minchisquare):
EMGout = params
minchisquare = chisqr
if np.sqrt(minchisquare) / np.mean(data) < 0.01:
break
else:
print(
"solver wrong! specify one of these: pyipm, trust-constr, lmfit !"
)
return [False, False]
except:
print("Unsuccessful Fitting!")
continue
if minchisquare < 0:
return [False, False]
else:
return [EMGout, True]
def EMAFit(x, data, nSample, **kwargs):
maxtry = 100
if "maxtry" in kwargs:
maxtry = kwargs["maxtry"]
# initial guess of each parameter
nW = len(x)
ini_a = np.sum(
(data[1:] + data[0:nW - 1]) / 2. * np.absolute(x[1:] - x[0:nW - 1]))
maxxW = x[np.nanargmax(data)]
# if maxxW <= x[0]:
# return [False,False]
# Weights of each point
DtWeights = data - data + 1
# DtWeights[data > np.nanpercentile(data, 75.)] = 4.
# DtWeights[data < np.nanpercentile(data, 25.)] = 4.
HM = (np.max(data) - np.min(data)) / 2. + np.min(data)
FWHM = np.absolute(x[np.nanargmin(np.absolute(data - HM))] -
maxxW) * 2. / 2.355
minchisquare = -1
#resx = np.absolute(x[1] - x[0])
means = np.array(
[ini_a / 2. / 50., ini_a / 2. / 10., 50., 10., FWHM, FWHM / 5.])
parnames = ['a', 'c', 'xa', 'xc', 'xsca', 'xscc', 'sigmaa', 'sigmac', 'b']
if 'fixparam' in kwargs:
fixpars = kwargs['fixparam']
fixvals = kwargs['fixvalue']
if 'consparam' in kwargs:
conspars = kwargs['consparam']
consvalsl = kwargs['consvaluel']
consvalsu = kwargs['consvalueu']
# DoSameSource = False
# if 'sameSource' in kwargs:
# if kwargs['sameSource'] == True:
# DoSameSource = True
PDom = False #no precursor source
SDom = False #no primary source
SameSource = False #source location is the same for aerosols and precursors
SameSigma = False
if 'sameSource' in kwargs:
if kwargs['sameSource'] == True:
SameSource = True
if 'sameSigma' in kwargs:
if kwargs['sameSigma'] == True:
SameSigma = True
if 'pDom' in kwargs:
if kwargs['pDom'] == True:
PDom = True
if 'sDom' in kwargs:
if kwargs['sDom'] == True:
SDom = True
if 'solver' in kwargs:
solver = kwargs['solver']
else:
solver = 'trust-constr'
FoundSolution = False
ntry = 0
SampleDoubled = False
while (FoundSolution == False) & (ntry < maxtry):
ntry = ntry + 1
SolutionUpdated = False
design = lhs(6, samples=nSample)
stds = 0.99 * means
for i in np.arange(6):
design[:, i] = norm(loc=means[i], scale=stds[i]).ppf(design[:, i])
#solution for one sets of initial guess
for jfit in np.arange(nSample + 1):
if jfit < nSample:
fitpars = design[jfit, :]
else:
fitpars = means
if np.min(fitpars) < 0:
continue
if solver == 'lmfit':
params = Parameters()
params.add('a', value=fitpars[0],
min=0.) # x-direction integration of total burden
params.add('xa', value=fitpars[2], min=1. / 3)
params.add('xc', value=fitpars[3], min=1. / 3)
# params.add('fc', value=inifc, min=0., max=1.)
params.add('c', value=fitpars[1], min=0.)
params.add('xsca', value=maxxW, min=np.min(x), max=np.max(x))
params.add('xscc', value=maxxW, min=np.min(x), max=np.max(x))
params.add('sigmaa',
value=fitpars[4],
min=0.5 / samplewd,
max=np.max([maxxW - np.min(x),
np.max(x) - maxxW
])) # standard deviation
params.add('sigmac',
value=fitpars[5],
min=0.5 / samplewd,
max=np.max([maxxW - np.min(x),
np.max(x) - maxxW
])) # standard deviation
params.add('b', value=np.min(data), min=0.,
max=np.max(data)) # background concentration
out = minimize(residualEMA,
params,
args=(x, data),
iter_cb=EMACall)
chisqr = np.sum(residualEMA(out.params, x, data, DtWeights)**2)
if (minchisquare < 0) | (chisqr < minchisquare):
EMAout = out.params
minchisquare = chisqr
if np.sqrt(minchisquare) / np.mean(data) < 0.01:
break
elif solver == 'trust-constr': # Do the scipy.optimize fit using "trust_constr" method
global ukninds, kninds, knvals
ukninds = np.arange(9).astype(int)
kninds = np.array([]).astype(int)
knvals = []
x0 = np.zeros(9)
x0[0:4] = fitpars[0:4]
x0[6:8] = fitpars[4:6]
x0[8] = np.min(data)
x0[4] = maxxW # , min = x[0], max = x[-1])
x0[5] = maxxW # , min = x[0], max = x[-1])
lowbs = np.array(
[0., 0., 0., 0.,
np.min(x),
np.min(x), 0., 0., 0.])
highbs = np.array([
np.inf, np.inf, np.inf, np.inf,
np.max(x),
np.max(x), np.inf, np.inf,
np.max(data)
]) #np.max([maxxW - np.min(x), np.max(x) - maxxW])
if 'consparam' in kwargs:
for ipar in np.arange(len(consvalsl)):
lowbs[conspars[ipar]] = consvalsl[ipar]
highbs[conspars[ipar]] = consvalsu[ipar]
x0[conspars[ipar]] = np.mean(
[consvalsl[ipar], consvalsu[ipar]])
# test if solutions are similar for different "fixsource" handling
if PDom == True:
kninds = np.append(kninds,
np.array([1, 3, 5, 7]).astype(int))
knvals = np.append(knvals, [0., 0., 0., 0.])
delinds = ~np.isin(ukninds, [1, 3, 5, 7])
ukninds = ukninds[delinds]
x0 = x0[delinds]
if SDom == True:
kninds = np.append(kninds, np.array([0, 4, 6]).astype(int))
knvals = np.append(knvals, [0., 0., 0.])
delinds = ~np.isin(ukninds, [0, 4, 6])
ukninds = ukninds[delinds]
x0 = x0[delinds]
if 'fixparam' in kwargs:
for ipar in np.arange(len(fixvals)):
if ~np.isin(fixpars[ipar], ukninds):
continue
kninds = np.append(kninds, np.int(fixpars[ipar]))
knvals = np.append(knvals, fixvals[ipar])
delinds = ~np.isin(ukninds, fixpars[ipar])
ukninds = ukninds[delinds]
x0 = x0[delinds]
# process bounds to be consistent with x0 and ukninds...
bounds = Bounds(lowbs[ukninds],
highbs[ukninds],
keep_feasible=True)
lconstmat = [[]]
lconstmin = []
lconstmax = []
if SameSource == True:
if ~np.isin(4, ukninds) & ~np.isin(5, ukninds):
lconstmat = np.append(lconstmat,
[np.zeros(len(ukninds))],
axis=0)
lconstmat[:, ukninds == 4] = 1
lconstmat[:, ukninds == 5] = 1
lconstmin = np.append(lconstmin, 0.)
lconstmax = np.append(lconstmax, 0.)
if SameSigma == True:
if ~np.isin(6, ukninds) & ~np.isin(7, ukninds):
lconstmat = np.append(lconstmat,
[np.zeros(len(ukninds))],
axis=0)
lconstmat[:, ukninds == 6] = 1
lconstmat[:, ukninds == 7] = 1
lconstmin = np.append(lconstmin, 0.)
lconstmax = np.append(lconstmax, 0.)
nlconstr = NonlinearConstraint(EMANLConstr1, -np.inf, 0.)
if (SameSource == True) | (SameSigma == True):
lconstr = LinearConstraint(lconstmat, lconstmin, lconstmax)
constraints = [lconstr, nlconstr]
else:
constraints = nlconstr
try:
res = scipyminimize(EMAchisqr1, x0, args=(x, data, DtWeights), method='trust-constr', \
constraints=constraints,options={'verbose': 0,'maxiter':50000}, bounds=bounds) #constraints=constraints,
except:
continue
pars = res.x
Hess = nd.Hessian(EMAchisqr1)(pars, x, data, DtWeights)
# args = {'x0': x[0]}
# uncertanties from fitting, following Laughner et al. (2016)
chisqr = EMAchisqr1(pars, x, data, DtWeights)
try:
invHess = np.linalg.inv(Hess)
except:
invHess = np.linalg.pinv(Hess)
if np.any(np.isnan(invHess) == True):
invHess = np.linalg.pinv(Hess)
if np.any(np.isnan(invHess) == True):
continue
sigmas = np.sqrt(invHess * chisqr / (data.size - pars.size))
params = Parameters()
for ikn in np.arange(len(kninds)):
params.add(parnames[kninds[ikn]],
value=knvals[ikn],
brute_step=0.)
for iukn in np.arange(len(ukninds)):
params.add(parnames[ukninds[iukn]],
value=pars[iukn],
brute_step=sigmas[iukn, iukn])
if (minchisquare < 0) | (chisqr < minchisquare):
SolutionUpdated = True
EMAout = params
minchisquare = chisqr
# update initial guess
means = np.array([EMAout['a'].value, EMAout['c'].value, EMAout['xa'].value, EMAout['xc'].value,\
EMAout['sigmaa'].value, EMAout['sigmac'].value])
if minchisquare < 0:
continue
if np.sqrt(minchisquare) / np.mean(data) < 0.01:
FoundSolution = True
ntry = maxtry
break
else:
print(
"solver wrong! specify one of these: pyipm, trust-constr, lmfit !"
)
return [False, False]
if (SolutionUpdated == False) & (SampleDoubled == False):
nSample = np.int(nSample * 2)
SampleDoubled = True
continue
if (SolutionUpdated == True) & (SampleDoubled == True):
nSample = np.int(nSample / 2)
SampleDoubled = False
continue
if (SolutionUpdated == False) & (SampleDoubled == True):
nSample = np.int(nSample / 2)
ntry = maxtry
SampleDoubled = False
continue
if minchisquare < 0:
return [False, False]
else:
return [EMAout, True]
def EMGFit(x, data, nSample, **kwargs):
maxtry = 100
if "maxtry" in kwargs:
maxtry = kwargs["maxtry"]
parnames = ['a', 'x0', 'xsc', 'sigma', 'b']
# initial guess of each parameter
nW = len(x)
ini_a = np.sum(
(data[1:] + data[0:nW - 1]) / 2. * np.absolute(x[1:] - x[0:nW - 1]))
#Weights of each point
DtWeights = data - data + 1
# DtWeights[data>np.nanpercentile(data,75.)]=4.
# DtWeights[data < np.nanpercentile(data, 25.)] = 4.
maxxW = x[np.nanargmax(data)]
# print maxxW,x[0]
# if maxxW <= x[0]:
# return [False,False]
#resx=np.absolute(x[1]-x[0])
HM = (np.max(data) - np.min(data)) / 2. + np.min(data)
FWHM = np.absolute(x[np.nanargmin(np.absolute(data - HM))] -
maxxW) * 2. / 2.355
minchisquare = -1
means = np.array([ini_a, 50., FWHM])
Gaussian = False
GauDecay = False
if 'GauDecay' in kwargs:
if kwargs['GauDecay'] == True:
GauDecay = True
avgw10 = kwargs['minx0'] / 3600.
if 'fixparam' in kwargs:
fixpars = kwargs['fixparam']
fixvals = kwargs['fixvalue']
if 'solver' in kwargs:
solver = kwargs['solver']
else:
solver = 'trust-constr'
FoundSolution = False
SampleDoubled = False
ntry = 0
while (FoundSolution == False) & (ntry < maxtry):
ntry = ntry + 1
SolutionUpdated = False
design = lhs(3, samples=nSample)
stds = 0.99 * means
for i in np.arange(3):
design[:, i] = norm(loc=means[i], scale=stds[i]).ppf(design[:, i])
for jfit in np.arange(nSample + 1):
if jfit < nSample:
fitpars = design[jfit, :]
else:
fitpars = means
if np.min(fitpars) < 0:
continue
if solver == 'lmfit':
params = Parameters()
params.add('a', value=fitpars[0],
min=0.) # x-direction integration of total burden
params.add('x0', value=fitpars[1], min=resx /
3) # x-direction distance in one-lifetime
params.add('xsc', value=maxxW, min=np.min(x), max=np.max(x))
params.add('sigma',
value=fitpars[2],
min=0.5,
max=np.max([maxxW - np.min(x),
np.max(x) - maxxW
])) # # standard deviation
params.add('b', value=np.min(data), min=0.,
max=np.max(data)) # background concentration
out = minimize(residualEMG,
params,
args=(x, data),
iter_cb=EMGCall)
chisqr = np.sum(residualEMG(out.params, x, data)**2)
if (minchisquare < 0) | (chisqr < minchisquare):
EMGout = out.params
minchisquare = chisqr
if np.sqrt(minchisquare) / np.mean(data) < 0.01:
break
elif solver == 'trust-constr': # Do the scipy.optimize fit using "trust_constr" method
global uknindsg, knindsg, knvalsg
uknindsg = np.arange(5).astype(int)
knindsg = np.array([]).astype(int)
knvalsg = []
x0 = np.zeros(5)
x0[0:2] = fitpars[0:2]
x0[3] = fitpars[2]
x0[2] = maxxW # , min = x[0], max = x[-1])
x0[4] = np.min(data)
lbds = np.array([0., 0., np.min(x), 0., 0.])
hbds = np.array([
np.inf, np.inf,
np.max(x), np.inf,
np.max(data)
]) #np.max([maxxW - np.min(x), np.max(x) - maxxW])
# test if solutions are similar for different "fixsource" handling
if 'fixparam' in kwargs:
for ipar in np.arange(len(fixvals)):
if (fixpars[ipar]
== 1) & (np.absolute(fixvals[ipar]) <= 1.e-20):
Gaussian = True
if ~np.isin(fixpars[ipar], uknindsg):
continue
knindsg = np.append(knindsg, np.int(fixpars[ipar]))
knvalsg = np.append(knvalsg, fixvals[ipar])
delinds = ~np.isin(uknindsg, fixpars[ipar])
uknindsg = uknindsg[delinds]
x0 = x0[delinds]
# process bounds to be consistent with x0 and ukninds...
bounds = Bounds(lbds[uknindsg],
hbds[uknindsg],
keep_feasible=True)
nlconstr = NonlinearConstraint(EMGNLConstr1, -np.inf, 100.)
try:
if Gaussian == True:
res = scipyminimize(EMGchisqr2, x0, args=(x, data, DtWeights), method='trust-constr', \
options={'verbose': 0 ,'maxiter':50000}, \
bounds=bounds) # constraints=[lconstr, nlconstr],
Hess = nd.Hessian(EMGchisqr2)(res.x, x, data)
pars = res.x
chisqr = EMGchisqr2(pars, x, data)
else:
res = scipyminimize(EMGchisqr1, x0, args=(x, data, DtWeights), method='trust-constr', \
constraints=[nlconstr],options={'verbose': 0,'maxiter':50000}, \
bounds=bounds) # constraints=[nlconstr],
Hess = nd.Hessian(EMGchisqr1)(res.x, x, data,
DtWeights)
pars = res.x
chisqr = EMGchisqr1(pars, x, data, DtWeights)
except:
continue
try:
invHess = np.linalg.inv(Hess)
except:
invHess = np.linalg.pinv(Hess)
if np.any(np.isnan(invHess) == True):
invHess = np.linalg.pinv(Hess)
if np.any(np.isnan(invHess) == True):
continue
sigmas = np.sqrt(invHess * chisqr / (data.size - pars.size))
params = Parameters()
for ikn in np.arange(len(knindsg)):
params.add(parnames[knindsg[ikn]],
value=knvalsg[ikn],
brute_step=0.)
for iukn in np.arange(len(uknindsg)):
params.add(parnames[uknindsg[iukn]],
value=pars[iukn],
brute_step=sigmas[iukn, iukn])
if (minchisquare < 0) | (chisqr < minchisquare):
SolutionUpdated = True
EMGout = params
minchisquare = chisqr
# update initial guess
means = np.array([
EMGout['a'].value, EMGout['x0'].value,
EMGout['sigma'].value
])
if np.sqrt(minchisquare) / np.mean(data) < 0.01:
FoundSolution = True
ntry = maxtry
break
else:
print(
"solver wrong! specify one of these: pyipm, trust-constr, lmfit !"
)
return [False, False]
if (SolutionUpdated == False) & (SampleDoubled == False):
nSample = np.int(nSample * 2)
SampleDoubled = True
continue
if (SolutionUpdated == True) & (SampleDoubled == True):
nSample = np.int(nSample / 2)
SampleDoubled = False
continue
if (SolutionUpdated == False) & (SampleDoubled == True):
nSample = np.int(nSample / 2)
ntry = maxtry
SampleDoubled = False
continue
if minchisquare < 0:
return [False, False]
else:
return [EMGout, True]
return (multi_obj_acq_total / sample_number)
# l-bfgs-b acquisation optimization
x_tries = np.random.uniform(bound[0], bound[1], size=(1000, d))
y_tries = [mesmo_acq(x) for x in x_tries]
sorted_indecies = np.argsort(y_tries)
i = 0
x_best = x_tries[sorted_indecies[i]]
while (any((x_best == x).all() for x in GPs[0].xValues)):
i = i + 1
x_best = x_tries[sorted_indecies[i]]
y_best = y_tries[sorted_indecies[i]]
x_seed = list(np.random.uniform(low=bound[0], high=bound[1], size=(1000, d)))
for x_try in x_seed:
result = scipyminimize(mesmo_acq, x0=np.asarray(x_try).reshape(1, -1), method='L-BFGS-B', bounds=Fun_bounds)
if not result.success:
continue
if ((result.fun <= y_best) and (not (result.x in np.asarray(GPs[0].xValues)))):
x_best = result.x
y_best = result.fun
# ---------------Updating and fitting the GPs-----------------
for i in range(len(functions)):
GPs[i].addSample(x_best, functions[i](list(x_best), d))
GPs[i].fitModel()
############################ write Input output into file ##################
input_output = open(os.path.join(paths, 'input_output.txt'), "a")
input_output.write(
str(GPs[0].xValues[-1]) + '---' + str([GPs[i].yValues[-1] for i in range(len(functions))]) + '\n')