def computeOutput(ir, ic, Ide, rf, nx, ny, Prec, rad, Indic, distanceMat):
print(ir, ic)
nSc = Ide.shape[0] # -1;# size(Ide,2)-1
x0 = [0, 0, 0, 0, 0, 1.5, 1.5, 1.5, 1.5, 1.5]
# t1=t.time();
inp0 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide,
0], rad)
# patches
inp1 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide,
1], rad)
# patches
inp2 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide,
2], rad)
# patches
inp3 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide,
3], rad)
# patches
inp4 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide,
4], rad)
# patches
# print("1: %s" % (t.time() - t1));
# t1=t.time();
out0 = ei.EquaIndic(ic, ir, rf, nx, ny, nSc, Indic[:, :, Ide])
# indicator
out0 = out0.flatten()
# print("2: %s" % (t.time() - t1));
# remove 0 elements
idxToRemove = np.where(out0 == 0)
out0 = np.delete(out0, idxToRemove)
inp0 = np.delete(inp0, idxToRemove, axis=0)
inp1 = np.delete(inp1, idxToRemove, axis=0)
inp2 = np.delete(inp2, idxToRemove, axis=0)
inp3 = np.delete(inp3, idxToRemove, axis=0)
inp4 = np.delete(inp4, idxToRemove, axis=0)
#
bnds1 = tuple((0, 5) for n in range(0, 5))
bnds2 = tuple((1, 3) for n in range(0, 5))
bnds = bnds1 + bnds2
mdl = minimize(iop,
x0,
args=(inp0, inp1, inp2, inp3, inp4, out0, distanceMat),
bounds=bnds,
method='SLSQP',
options={
'maxiter': 500,
'ftol': 0.001
})
omega = mdl.x[5:]
alpha = mdl.x[0:5]
return mdl.x
def computeOutput(ir, ic, Ide, rf, nx, ny, Prec, PREC, Indic, ICvecalpha,
ICvecomega, rad, poly, lat):
print(ir, ic)
ratio = np.polyval(poly, lat[ir, ic])
# print('step1')
# print(lat[ir, ic], ratio)
Y, X = np.mgrid[-rad:rad + 1:1, -rad:rad + 1:1]
F = 1 / ((1 + ((X / ratio)**2 + Y**2)**0.5))
nSc = Ide.shape[0] - 1
# size(Ide,2)-1
inp1 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3],
Prec[:, :, Ide[1], PREC], rad)
# patches
inp2 = ei.EquaIndic(ic, ir, rf, nx, ny, nSc, Indic[:, :, Ide[1]])
# indicator
# print(inp1.shape)
remInd = (inp2 > 0).flatten()
inp1 = inp1[remInd] # [ind,:];
inp2 = inp2[remInd] # [ind];
x0tmp_alpha = ICvecalpha[PREC]
x0tmp_omega = ICvecomega[PREC]
x0 = [x0tmp_alpha, x0tmp_omega]
bndstmp_alpha = (0, 10) # x0tmp_alpha - 0.5, x0tmp_alpha + 0.5)
bndstmp_omega = (x0tmp_omega - 0.5, x0tmp_omega + 5.5)
# bnds = ((0, 2), bndstmp)
bnds = (bndstmp_alpha, bndstmp_omega)
# a=time.time()
opts = {'disp': False, 'ftol': 10e-6}
mdl = minimize(iop2,
x0,
args=(inp1, inp2, rad, F),
bounds=bnds,
method='Nelder-Mead',
options=opts) # L-BFGS-B, COBYLA SLSQP
# print(mdl.x)
# print(mdl.success)
return mdl.x
def step1_omegaOptPerGroupOfCells_iniRes(conf):
#convert from 28 to 7 km
Prec = conf.Prec
ny = conf.ny
nx = conf.nx
rad = conf.radStep1
nPrec = conf.nPrec
rf = conf.rf
flagRegioMat = np.copy(conf.flagRegioMat)
step = conf.stepOptPerGroupCells
#pad Prec with zeros around initial matrix, to perform matrix products later on
Prec2 = np.zeros(
(ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3]))
Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :]
Prec = Prec2
#convert from 28 to 7 km
Indic = conf.Indic
flagRegioMat = flagRegioMat
#initialize variables
omega = np.full([ny, nx, nPrec], np.nan)
# alpha = np.full([ny,nx,nPrec],np.nan);
#define training scenarios; note scenarios number is +1 if checking DoE...as in line 74 it is -1
if conf.domain == 'emep10km':
if conf.aqi == 'SURF_ug_PM25_rh50-Yea':
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 5]), np.array([1, 6]))
elif conf.aqi == 'SURF_ug_PM10_rh50-Yea':
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 4]), np.array([1, 6]))
elif conf.domain == 'ineris7km':
IdeVec = (np.array([1, 2]), np.array([1, 3]), np.array([1, 4]),
np.array([1, 5]), np.array([1, 6]))
#loop over precursors
for precursor in range(0, nPrec):
PREC = precursor
Ide = IdeVec[precursor]
#intialize variables
print('precursor: ' + str(PREC))
# icel=1
# flagIC = 0
for ic in range(0, nx, step):
for ir in range(0, ny, step):
# print(PREC, ic);
icel = 0
PrecPatch = np.zeros((step * step, (rad * 2 + 1)**2))
IndicEq = np.zeros((step * step, 1))
print(PREC, ir, ic)
for icc in range(ic, ic + step):
for irr in range(ir, ir + step):
if (icc < nx) and (irr <
ny) and (flagRegioMat[ir, ic]) > 0:
nSc = Ide.shape[0] - 1
# size(Ide,2)-1
tmpPrec = ep.EquaPrec(icc, irr, rf, nx, ny, nSc,
Prec.shape[3],
Prec[:, :, Ide[1],
PREC], rad)
# patches
tmpInde = ei.EquaIndic(icc, irr, rf, nx, ny, nSc,
Indic[:, :, Ide[1]])
# indicator
# print(inp1.shape)
PrecPatch[icel, :] = tmpPrec
#np.squeeze(tmpPrec)
IndicEq[icel] = tmpInde
icel = icel + 1
remInd = (IndicEq > 0).flatten()
x0 = [1, 1.5]
bnds = ((0, 2), (0.1, 2.9))
inp1 = PrecPatch[remInd] # [ind,:];
inp2 = IndicEq[remInd] # [ind];
if inp2.size == 0:
omega[ir:ir + step, ic:ic + step,
PREC] = conf.omegaFinalStep1_28km[irr - step,
icc - step, PREC]
else:
# scaler = MinMaxScaler(feature_range=(0, 1))
# scaler.fit(inp1)
# inp1 = scaler.transform(inp1)
opts = {'disp': False, 'ftol': 10e-9}
mdl = minimize(iop,
x0,
args=(inp1, inp2, rad),
bounds=bnds,
method='L-BFGS-B',
options=opts) #L-BFGS-B, COBYLA SLSQP
if (mdl.success
== True) and (mdl.x[1] > 0.1) and (mdl.x[1] < 2.9):
omega[ir:ir + step, ic:ic + step, PREC] = mdl.x[1]
#mdl[1];
else:
omega[ir:ir + step, ic:ic + step,
PREC] = conf.omegaFinalStep1_28km[irr - step,
icc - step,
PREC]
### if results < 1, interpolate
# for i in range(0,5):
# val=omega[:,:,i]
# val[val<=0.1]=np.nan
# x = np.arange(0, val.shape[1])
# y = np.arange(0, val.shape[0])
# val = np.ma.masked_invalid(val)
# xx, yy = np.meshgrid(x, y)
# #get only the valid values
# x1 = xx[~val.mask]
# y1 = yy[~val.mask]
# newarr = val[~val.mask]
# GD1 = interpol.griddata((x1, y1), newarr.ravel(),(xx, yy),method='cubic')
## GD1 = gf.filters.gaussian_filter(val, 1, order=0)
# omega[:,:,i]=GD1
#store final results
# for i in range(0,5):
# tmp = omega[:,:,i]
# tmp = gf.gaussian_filter(tmp, sigma=10)
# omega[:, :, i] = tmp
omegaFinal = q.quant(omega, 0.1) #discretize
conf.omegaFinalStep1 = omegaFinal
conf.ci2Step1 = []
conf.CovB2Step1 = []
def step1_potency(conf):
prctileVec1 = np.array([35, 35, 35, 35, 35])
prctileVec2 = np.array([70, 70, 70, 70, 70])
categories = np.array([1, 2, 3])
#convert from 28 to 7 km
Prec = f7.from7to28(conf.Prec)
ny = int(conf.ny / 4)
nx = int(conf.nx / 4)
rad = conf.radStep1
nPrec = conf.nPrec
rf = conf.rf
flagRegioMat = np.copy(conf.flagRegioMat)
#pad Prec with zeros around initial matrix, to perform matrix products later on
Prec2 = np.zeros(
(ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3]))
Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :]
Prec = Prec2
#convert from 28 to 7 km
Indic = f7.from7to28(conf.Indic)
flagRegioMat = f7.from7to28(flagRegioMat)
#initialize variables
omega = np.full([ny, nx, nPrec], np.nan)
alpha = np.full([ny, nx, nPrec], np.nan)
ci2 = np.empty((categories.size, nPrec), dtype=object)
CovB2 = np.empty((categories.size, nPrec), dtype=object)
alphaTmp = np.zeros((categories.size))
omegaTmp = np.zeros((categories.size))
#define training scenarios; note scenarios number is +1 if checking DoE...as in line 74 it is -1
if conf.domain == 'emep10km':
if conf.aqi == 'SURF_ug_PM25_rh50-Yea':
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 5]), np.array([1, 6]))
elif conf.aqi == 'SURF_ug_PM10_rh50-Yea':
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 4]), np.array([1, 6]))
elif conf.domain == 'ineris7km':
IdeVec = (np.array([1, 8]), np.array([1, 9]), np.array([1, 10]),
np.array([1, 11]), np.array([1, 12]))
#loop over precursors
for precursor in range(0, nPrec):
PREC = precursor
Ide = IdeVec[precursor]
icel = 0
#intialize variables
PrecPatch = np.zeros((nx * ny, (rad * 2 + 1)**2))
IndicEq = np.zeros((nx * ny, 1))
indexUsed = np.full((nx * ny, 1), np.nan)
#np.zeros((nx*ny,1));
potency = np.full((ny, nx), np.nan)
#np.zeros((ny,nx));
print('precursor: ' + str(PREC))
#loop over cells to create groups
for ic in range(0, nx):
#print(PREC, ic);
for ir in range(0, ny):
if flagRegioMat[ir, ic] > 0:
#create data for omega calculation
nSc = Ide.shape[0] - 1
# size(Ide,2)-1
tmpPrec = ep.EquaPrec(ic, ir, rf, nx, ny, nSc,
Prec.shape[3], Prec[:, :, Ide[1],
PREC], rad)
# patches
tmpInde = ei.EquaIndic(ic, ir, rf, nx, ny, nSc,
Indic[:, :, Ide[1]])
# indicator
x0 = np.array([1, 2])
[inp2_aggemi] = inv.InvDistN_opt_prec(x0, tmpPrec, rad)
#store data for omega calculation
potency[ir, ic] = tmpInde / inp2_aggemi
prc1 = np.percentile(potency[np.isfinite(potency)],
prctileVec1[precursor])
prc9 = np.percentile(potency[np.isfinite(potency)],
prctileVec2[precursor])
speed = potency.copy()
speed[np.isnan(speed)] = 0
potency[speed < prc1] = 1
potency[(speed >= prc1) & (speed < prc9)] = 2
potency[speed >= prc9] = 3
val = categories
for ic in range(0, nx):
#print(PREC, ic);
for ir in range(0, ny):
if flagRegioMat[ir, ic] > 0:
#variable to store which group ot be considered
indexUsed[icel] = np.where(val == potency[ir, ic])
#create data for omega calculation
nSc = Ide.shape[0] - 1
# size(Ide,2)-1
tmpPrec = ep.EquaPrec(ic, ir, rf, nx, ny, nSc,
Prec.shape[3], Prec[:, :, Ide[1],
PREC], rad)
# patches
tmpInde = ei.EquaIndic(ic, ir, rf, nx, ny, nSc,
Indic[:, :, Ide[1]])
# indicator
#store data for omega calculation
PrecPatch[icel, :] = tmpPrec
#np.squeeze(tmpPrec)
IndicEq[icel] = tmpInde
icel = icel + 1
indexUsedLin = np.reshape(indexUsed, -1, order='F')
#compute omega for each group of cells, given precursor p
for i in range(val.size):
x0 = [1, 2]
ind = np.where(indexUsedLin == i)[0]
inp1 = PrecPatch[ind, :]
inp2 = IndicEq[ind]
iop = lambda inp1, beta1, beta2: inv.InvDistN_opt_prec(
[beta1, beta2], inp1, rad)
[mdl, r, J, CovB] = nlin.nlinfit(iop, inp1, inp2.ravel(), x0)
ci2[i, PREC] = nlpa.nlparci(r, J)
CovB2[i, PREC] = CovB
alphaTmp[i] = mdl[0]
omegaTmp[i] = mdl[1]
#repeat result for each belonging to a given group
for ic in range(0, nx):
for ir in range(0, ny):
if flagRegioMat[ir, ic] > 0:
indexUsed = np.where(val == potency[ir, ic])[0]
alpha[ir, ic, PREC] = alphaTmp[indexUsed]
omega[ir, ic, PREC] = omegaTmp[indexUsed]
del (PrecPatch, IndicEq, indexUsed, potency, speed)
#rescale to initial spatial resolution, through nearest interpolation
#initialize variable
omegaFinal = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5))
#loop on precursors
for i in range(0, nPrec):
#define interpolator object
xgv = np.arange(1., conf.Prec.shape[0] / 4 + 1)
ygv = np.arange(1., conf.Prec.shape[1] / 4 + 1)
F = interpol.RegularGridInterpolator((xgv, ygv),
omega[:, :, i],
method='nearest',
bounds_error=False,
fill_value=None)
#interpolate
Xq = np.arange(1., conf.Prec.shape[0] / 4 + 1, 1 / 4)
Yq = np.arange(1., conf.Prec.shape[1] / 4 + 1, 1 / 4)
[Y2, X2] = np.meshgrid(Yq, Xq)
pts = ((X2.flatten(), Y2.flatten()))
omegaFinal[:, :, i] = F(pts).reshape(conf.Prec.shape[0],
conf.Prec.shape[1])
print('precursor interpolated: ' + str(i))
#store final results
conf.omegaFinalStep1 = omegaFinal
conf.ci2Step1 = ci2
conf.CovB2Step1 = CovB2
def step1_omegaOptimization(conf):
#convert from 28 to 7 km
# Prec = conf.Prec;
# ny = conf.ny;
# nx = conf.nx;
Prec = f7.from7to28(conf.Prec)
ny = int(conf.ny / 4)
nx = int(conf.nx / 4)
rad = conf.radStep1
nPrec = conf.nPrec
rf = conf.rf1
flagRegioMat = np.copy(conf.flagRegioMat)
#pad Prec with zeros around initial matrix, to perform matrix products later on
Prec2 = np.zeros(
(ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3]))
Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :]
Prec = Prec2
#convert from 28 to 7 km
# Indic = conf.Indic;
# flagRegioMat = flagRegioMat;
Indic = f7.from7to28(conf.Indic)
flagRegioMat = f7.from7to28(flagRegioMat)
lat = f7.from7to28(conf.y)
#initialize variables
omega = np.full([ny, nx, nPrec], np.nan)
alpha = np.full([ny, nx, nPrec], np.nan)
#precomputed vars
Y, X = np.mgrid[-rad:rad + 1:1, -rad:rad + 1:1]
distanceMat = 1 / ((1 + (X**2 + Y**2)**(0.5)))
distanceMat = distanceMat.flatten()
#define training scenarios; note scenarios number is +1 if checking DoE...as in line 74 it is -1
if conf.domain == 'emep10km':
if conf.aqi == 'SURF_ug_PM25_rh50-Yea':
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 5]), np.array([1, 6]))
elif conf.aqi == 'SURF_ug_PM10_rh50-Yea':
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 4]), np.array([1, 6]))
elif conf.domain == 'ineris7km':
IdeVec = (np.array([1, 2]), np.array([1, 3]), np.array([1, 4]),
np.array([1, 5]), np.array([1, 6]))
elif (conf.domain == 'emepV433_camsV221') | (
conf.domain == 'edgar2015') | (conf.domain == 'emepV434_camsV42'):
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 4]), np.array([1, 5]))
for precursor in range(0, nPrec):
Ide = conf.Ide
PREC = precursor
Ide = IdeVec[precursor]
for ic in range(0, nx):
print(ic, nx)
for ir in range(0, ny):
if flagRegioMat[ir, ic] > 0:
# ratio = np.polyval(conf.ratioPoly, conf.y[ir, ic])
ratio = np.polyval(conf.ratioPoly, lat[ir, ic])
Y, X = np.mgrid[-rad:rad + 1:1, -rad:rad + 1:1]
distanceMat = 1 / ((1 + ((X / ratio)**2 + Y**2)**0.5))
distanceMat = distanceMat.flatten()
# ratio = np.polyval(conf.ratioPoly, lat[ir, ic])
# Y, X = np.mgrid[-rad:rad + 1:1, -rad:rad + 1:1];
# F = 1 / ((1 + ((X / ratio) ** 2 + Y ** 2) ** 0.5));
#
# Y, X = np.mgrid[-rad:rad+1:1, -rad:rad+1:1];
# distanceMat = 1 / ( (1 + (X**2 + Y**2)**(0.5)))
# distanceMat = distanceMat.flatten()
x0 = [1, 1, 1, 1, 1, 1.5, 1.5, 1.5, 1.5, 1.5]
nSc = Ide.shape[0] #-1;# size(Ide,2)-1
# t1=t.time();
inp0 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3],
Prec[:, :, Ide, 0], rad)
# patches
inp1 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3],
Prec[:, :, Ide, 1], rad)
# patches
inp2 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3],
Prec[:, :, Ide, 2], rad)
# patches
inp3 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3],
Prec[:, :, Ide, 3], rad)
# patches
inp4 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3],
Prec[:, :, Ide, 4], rad)
# patches
# print("1: %s" % (t.time() - t1));
# t1=t.time();
out0 = ei.EquaIndic(ic, ir, rf, nx, ny, nSc, Indic[:, :,
Ide])
# indicator
out0 = out0.flatten()
# print("2: %s" % (t.time() - t1));
#remove 0 elements
idxToRemove = np.where(out0 == 0)
out0 = np.delete(out0, idxToRemove)
inp0 = np.delete(inp0, idxToRemove, axis=0)
inp1 = np.delete(inp1, idxToRemove, axis=0)
inp2 = np.delete(inp2, idxToRemove, axis=0)
inp3 = np.delete(inp3, idxToRemove, axis=0)
inp4 = np.delete(inp4, idxToRemove, axis=0)
bnds1 = tuple((0, 5) for n in range(0, 5))
bnds2 = tuple((1, 3) for n in range(0, 5))
bnds = bnds1 + bnds2
#DO NOT FIND OPTIMAL SOLUTION
# t1=t.time();
# mdl = minimize(iop, x0, args=(inp0, inp1, inp2, inp3, inp4, out0, distanceMat), method='Nelder-Mead', options={'maxiter':500})
# print (t.time()-t1)
# #OK
# t1=t.time();
# mdl = minimize(iop, x0, args=(inp0, inp1, inp2, inp3, inp4, out0, distanceMat), bounds=bnds, method='L-BFGS-B', options={'maxiter':500})
# print (t.time()-t1)
#
# t1=t.time();
# mdl = minimize(iop, x0, args=(inp0, inp1, inp2, inp3, inp4, out0, distanceMat), method='COBYLA', options={'maxiter':500})
# (t.time()-t1)
#
# t1=t.time();
mdl = minimize(iop,
x0,
args=(inp0, inp1, inp2, inp3, inp4, out0,
distanceMat),
bounds=bnds,
method='SLSQP',
options={'maxiter': 500})
# print(mdl)
# print("4: %s" % (t.time() - t1));
# print(mdl.x[5:])
omega[ir, ic, :] = mdl.x[5:]
alpha[ir, ic, :] = mdl.x[0:5]
conf.omegaFinalStep1_notFiltered = omega
omegaFinal2 = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5))
for i in range(0, nPrec):
for irAgg in range(0, ny):
for icAgg in range(0, nx):
omegaFinal2[irAgg * 4:irAgg * 4 + 4, icAgg * 4:icAgg * 4 + 4,
i] = omega[irAgg, icAgg, i]
print('precursor interpolated: ' + str(i))
# omegaFinal = omegaFinal2
omegaFinal = np.zeros_like(omegaFinal2)
for i in range(0, 5):
tmp = omegaFinal2[:, :, i]
tmp = gf.gaussian_filter(tmp, sigma=.5)
omegaFinal[:, :, i] = tmp
omegaFinal = np.round(omegaFinal, 1)
# omegaFinal = q.quant(omegaFinal, 0.2) # discretize
#keep only results on the mask
omegaFinal[conf.flagRegioMat == 0] = np.nan
conf.omegaFinalStep1 = omegaFinal
conf.ci2Step1 = []
conf.CovB2Step1 = []
def step1_omegaOptSliding_aggRes_perPoll(conf):
prctileVec1 = np.array([100, 100, 100, 100, 100])
# prctileVec2=np.array([70, 70, 70, 70, 70]);
categories = np.array([1])
#convert from 28 to 7 km
Prec = f7.from7to28(conf.Prec)
ny = int(conf.ny / 4)
nx = int(conf.nx / 4)
rad = conf.radStep1
nPrec = len(conf.vec3[conf.POLLSEL]) #conf.nPrec;
rf = 0 #conf.rf1;
flagRegioMat = np.copy(conf.flagRegioMat)
#pad Prec with zeros around initial matrix, to perform matrix products later on
Prec2 = np.zeros(
(ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3]))
Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :]
Prec = Prec2
#convert from 28 to 7 km
Indic = f7.from7to28(conf.Indic)
flagRegioMat = f7.from7to28(flagRegioMat)
#initialize variables
omega = np.full([ny, nx, nPrec], np.nan)
alpha = np.full([ny, nx, nPrec], np.nan)
ci2 = np.empty((nPrec), dtype=object)
CovB2 = np.empty((nPrec), dtype=object)
# alphaTmp = np.zeros((categories.size));
# omegaTmp = np.zeros((categories.size));
# define training scenarios; note scenarios number is +1 if checking DoE...as in line 74 it is -1
if conf.domain == 'emep10km':
# if conf.aqi == 'SURF_ug_PM25_rh50-Yea':
if 'SURF_ug_PM25_rh50' in conf.aqi:
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 5]), np.array([1, 6]))
# elif conf.aqi == 'SURF_ug_PM10_rh50-Yea':
elif 'SURF_ug_PM10_rh50' in conf.aqi:
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 4]), np.array([1, 6]))
elif 'SURF_ppb_O3' in conf.aqi:
IdeVec = (np.array([1, 1]), np.array([1, 2]))
elif 'SURF_ug_NOx' in conf.aqi:
IdeVec = (np.array([1, 1]), np.array([1, 2]))
elif conf.domain == 'ineris7km':
IdeVec = (np.array([1, 2]), np.array([1, 3]), np.array([1, 4]),
np.array([1, 5]), np.array([1, 6]))
#loop over precursors
for precursor in range(0, nPrec):
PREC = precursor
Ide = IdeVec[precursor]
icel = 0
#intialize variables
numcells = nx * ny
numcells = np.sum(
flagRegioMat >
0) # create empty matrix only for really needed points
PrecPatch = np.zeros((numcells, (rad * 2 + 1)**2))
IndicEq = np.zeros((numcells, 1))
indexUsed = np.full((numcells, 1), np.nan)
#np.zeros((nx*ny,1));
potency = np.full((numcells), np.nan)
#np.zeros((ny,nx));
print('precursor: ' + str(PREC))
for ic in range(0, nx):
#print(PREC, ic);
for ir in range(0, ny):
if flagRegioMat[ir, ic] > 0:
#variable to store which group ot be considered
# indexUsed[icel] = np.where(val==potency[ir,ic]);
#create data for omega calculation
nSc = Ide.shape[0] - 1
# size(Ide,2)-1
tmpPrec = ep.EquaPrec(ic, ir, rf, nx, ny, nSc,
Prec.shape[3], Prec[:, :, Ide[1],
PREC], rad)
# patches
tmpInde = ei.EquaIndic(ic, ir, rf, nx, ny, nSc,
Indic[:, :, Ide[1]])
# indicator
#store data for omega calculation
PrecPatch[icel, :] = tmpPrec
#np.squeeze(tmpPrec)
IndicEq[icel] = tmpInde
icel = icel + 1
# indexUsedLin = np.reshape(indexUsed, -1, order='F');
#compute omega for each group of cells, given precursor p
# for i in range(val.size):
remInd = (IndicEq > 0).flatten()
i = 1
x0 = [1, 1.5]
# ind = np.where(indexUsedLin==i)[0];
inp1 = PrecPatch[remInd] #[ind,:];
inp2 = IndicEq[remInd] #[ind];
iop = lambda inp1, beta1, beta2: inv.InvDistN_opt_prec([beta1, beta2],
inp1, rad)
[mdl, r, J, CovB] = nlin.nlinfit(iop, inp1, inp2.ravel(), x0)
# ci2[PREC] = nlpa.nlparci(r,J);
# CovB2[PREC] = CovB;
alphaTmp = mdl[0]
omegaTmp = mdl[1]
#repeat result for each belonging to a given group
for ic in range(0, nx):
for ir in range(0, ny):
if flagRegioMat[ir, ic] > 0:
# indexUsed = np.where(val==potency[ir,ic])[0];
alpha[ir, ic, PREC] = alphaTmp
omega[ir, ic, PREC] = omegaTmp
del (PrecPatch, IndicEq, indexUsed, potency)
#rescale to initial spatial resolution, through nearest interpolation
#initialize variable
omegaFinal = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5))
alphaFinal = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5))
[i, j] = np.where(np.isfinite(omega[:, :, 0]))
for p in range(0, nPrec):
omegaFinal[:, :, p] = omega[i[0], j[0], p]
alphaFinal[:, :, p] = alpha[i[0], j[0], p]
#store final results
conf.omegaFinalStep1_alldom = omegaFinal
conf.alphaFinalStep1_alldom = alphaFinal
conf.ci2Step1 = ci2
conf.CovB2Step1 = CovB2
#######################################################
#######################################################
#START NOW THE SECOND STEP OF THE COMPUTATION OF OMEGA
#######################################################
#######################################################
# convert from 28 to 7 km
Prec = f7.from7to28(conf.Prec)
ny = int(conf.ny / 4)
nx = int(conf.nx / 4)
rad = conf.radStep2
nPrec = conf.nPrec
rf = conf.rf1
flagRegioMat = np.copy(conf.flagRegioMat)
Y, X = np.mgrid[-rad:rad + 1:1, -rad:rad + 1:1]
# np.meshgrid(r_[-rad:1:rad], r_[-rad:1:rad]);
F = 1 / ((1 + (X**2 + Y**2)**(0.5)))
# pad Prec with zeros around initial matrix, to perform matrix products later on
Prec2 = np.zeros(
(ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3]))
Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :]
Prec = Prec2
# convert from 28 to 7 km
Indic = f7.from7to28(conf.Indic)
flagRegioMat = f7.from7to28(flagRegioMat)
af = conf.alphaFinalStep1_alldom
of = conf.omegaFinalStep1_alldom
ICvecalpha = [
af[np.isfinite(af)][0], af[np.isfinite(af)][1], af[np.isfinite(af)][2],
af[np.isfinite(af)][3], af[np.isfinite(af)][4]
]
ICvecomega = [
of[np.isfinite(of)][0], of[np.isfinite(of)][1], of[np.isfinite(of)][2],
of[np.isfinite(of)][3], of[np.isfinite(of)][4]
]
# initialize variables
omega = np.full([ny, nx, nPrec], 1.5)
# omega = np.full([ny, nx, nPrec], 2);
# for i in range(0,5):
# omega[:,:,i] = of[np.isfinite(of)][i]
# ICvecalpha = [1, 1, 1, 1, 1]
# ICvecomega = [1.5, 1.5, 1.5, 2.6, 1.5]
# loop over precursors
for precursor in range(0, nPrec):
PREC = precursor
Ide = IdeVec[precursor]
# intialize variables
print('precursor: ' + str(PREC))
# icel=1
# flagIC = 0
for ic in range(0, nx):
print(PREC, ic)
for ir in range(0, ny):
if flagRegioMat[ir, ic] > 0:
nSc = Ide.shape[0] - 1
# size(Ide,2)-1
inp1 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3],
Prec[:, :, Ide[1], PREC], rad)
# patches
inp2 = ei.EquaIndic(ic, ir, rf, nx, ny, nSc, Indic[:, :,
Ide[1]])
# indicator
# print(inp1.shape)
remInd = (inp2 > 0).flatten()
inp1 = inp1[remInd] # [ind,:];
inp2 = inp2[remInd] # [ind];
x0tmp_alpha = ICvecalpha[precursor]
x0tmp_omega = ICvecomega[precursor]
x0 = [x0tmp_alpha, x0tmp_omega]
bndstmp_alpha = (0, 10
) # x0tmp_alpha - 0.5, x0tmp_alpha + 0.5)
bndstmp_omega = (x0tmp_omega - 0.5, x0tmp_omega + 0.5)
# bnds = ((0, 2), bndstmp)
bnds = (bndstmp_alpha, bndstmp_omega)
# a=time.time()
opts = {'disp': False, 'ftol': 10e-6}
mdl = minimize(iop2,
x0,
args=(inp1, inp2, rad, F),
bounds=bnds,
method='L-BFGS-B',
options=opts) # L-BFGS-B, COBYLA SLSQP
# print(mdl.success)
omega[ir, ic, PREC] = mdl.x[1]
# mdl[1]
# if (mdl.x[1]<1.1) or (mdl.x[1]>1.9) :
# mdl.x[1] = ICvec[precursor]
# omega[ir, ic, PREC] = mdl.x[1]; # mdl[1]
# if (mdl.success == True) and (mdl.x[1] > 1) and (mdl.x[1] < 3) and (mdl.x[1] != x0[1]):
# conf.omegaAccuracy[ir,ic,PREC] = 2
# omega[ir, ic, PREC] = mdl.x[1]; # mdl[1];
# # print(mdl)
# else:
# omega[ir, ic, PREC] = conf.step1_omegaOptPerPoll_allCells[ir, ic , PREC]
# omega = np.round(omega,1)
conf.omegaFinalStep1_notFiltered = omega
omegaFinal2 = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5))
for i in range(0, nPrec):
for irAgg in range(0, ny):
for icAgg in range(0, nx):
omegaFinal2[irAgg * 4:irAgg * 4 + 4, icAgg * 4:icAgg * 4 + 4,
i] = omega[irAgg, icAgg, i]
print('precursor interpolated: ' + str(i))
# omegaFinal = omegaFinal2
omegaFinal = np.zeros_like(omegaFinal2)
for i in range(0, 5):
tmp = omegaFinal2[:, :, i]
tmp = gf.gaussian_filter(tmp, sigma=10)
omegaFinal[:, :, i] = tmp
omegaFinal = np.round(omegaFinal, 1)
# omegaFinal = q.quant(omegaFinal, 0.2) # discretize
conf.omegaFinalStep1 = omegaFinal
conf.ci2Step1 = []
conf.CovB2Step1 = []
def step1_omegaOptimization(conf):
prctileVec1 = np.array([100, 100, 100, 100, 100])
# prctileVec2=np.array([70, 70, 70, 70, 70]);
categories = np.array([1])
#convert from 28 to 7 km
Prec = f7.from7to28(conf.Prec)
ny = int(conf.ny / 4)
nx = int(conf.nx / 4)
rad = conf.radStep1
nPrec = len(conf.vec3[conf.POLLSEL]) #conf.nPrec;
rf = 0 #conf.rf1;
flagRegioMat = np.copy(conf.flagRegioMat)
#pad Prec with zeros around initial matrix, to perform matrix products later on
Prec2 = np.zeros(
(ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3]))
Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :]
Prec = Prec2
#convert from 28 to 7 km
Indic = f7.from7to28(conf.Indic)
flagRegioMat = f7.from7to28(flagRegioMat)
lat = f7.from7to28(conf.y)
#initialize variables
omega = np.full([ny, nx, nPrec], np.nan)
alpha = np.full([ny, nx, nPrec], np.nan)
ci2 = np.empty((nPrec), dtype=object)
CovB2 = np.empty((nPrec), dtype=object)
# alphaTmp = np.zeros((categories.size));
# omegaTmp = np.zeros((categories.size));
# define training scenarios; note scenarios number is +1 if checking DoE...as in line 74 it is -1
if conf.domain == 'emep10km':
# if conf.aqi == 'SURF_ug_PM25_rh50-Yea':
if 'SURF_ug_PM25_rh50' in conf.aqi:
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 5]), np.array([1, 6]))
# elif conf.aqi == 'SURF_ug_PM10_rh50-Yea':
elif 'SURF_ug_PM10_rh50' in conf.aqi:
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 4]), np.array([1, 6]))
elif 'SURF_ppb_O3' in conf.aqi:
IdeVec = (np.array([1, 1]), np.array([1, 2]))
elif 'SURF_ug_NOx' in conf.aqi:
IdeVec = (np.array([1, 1]), np.array([1, 2]))
elif conf.domain == 'ineris7km':
IdeVec = (np.array([1, 2]), np.array([1, 3]), np.array([1, 4]),
np.array([1, 5]), np.array([1, 6]))
#loop over precursors
for precursor in range(0, nPrec):
PREC = precursor
Ide = IdeVec[precursor]
icel = 0
#intialize variables
numcells = nx * ny
numcells = np.sum(
flagRegioMat >
0) # create empty matrix only for really needed points
PrecPatch = np.zeros((numcells, (rad * 2 + 1)**2))
IndicEq = np.zeros((numcells, 1))
latVec = np.zeros((numcells, 1))
indexUsed = np.full((numcells, 1), np.nan)
#np.zeros((nx*ny,1));
potency = np.full((numcells), np.nan)
#np.zeros((ny,nx));
print('precursor: ' + str(PREC))
for ic in range(0, nx):
#print(PREC, ic);
for ir in range(0, ny):
if flagRegioMat[ir, ic] > 0:
#variable to store which group ot be considered
# indexUsed[icel] = np.where(val==potency[ir,ic]);
#create data for omega calculation
nSc = Ide.shape[0] - 1
# size(Ide,2)-1
tmpPrec = ep.EquaPrec(ic, ir, rf, nx, ny, nSc,
Prec.shape[3], Prec[:, :, Ide[1],
PREC], rad)
# patches
tmpInde = ei.EquaIndic(ic, ir, rf, nx, ny, nSc,
Indic[:, :, Ide[1]])
# indicator
#store data for omega calculation
PrecPatch[icel, :] = tmpPrec
#np.squeeze(tmpPrec)
IndicEq[icel] = tmpInde
latVec[icel] = lat[ir, ic]
icel = icel + 1
# indexUsedLin = np.reshape(indexUsed, -1, order='F');
#compute omega for each group of cells, given precursor p
# for i in range(val.size):
remInd = (IndicEq > 0).flatten()
i = 1
x0 = [1, 1.5]
# ind = np.where(indexUsedLin==i)[0];
inp1 = PrecPatch[remInd] #[ind,:];
inp2 = IndicEq[remInd] #[ind];
latVecFilt = latVec[remInd]
bnds = ((0, 2), (0.1, 2.9))
opts = {'disp': False, 'ftol': 10e-9}
mdl = minimize(iop,
x0,
args=(inp1, inp2, rad, latVecFilt, conf.ratioPoly),
bounds=bnds,
method='L-BFGS-B',
options=opts) # L-BFGS-B, TNC
# iop = lambda inp1,beta1,beta2: inv.InvDistN_opt_prec([beta1,beta2],inp1,rad);
#
# [mdl,r,J,CovB] = nlin.nlinfit(iop,inp1,inp2.ravel(),x0);
# ci2[PREC] = nlpa.nlparci(r,J);
# CovB2[PREC] = CovB;
alphaTmp = mdl.x[0]
omegaTmp = mdl.x[1]
#repeat result for each belonging to a given group
for ic in range(0, nx):
for ir in range(0, ny):
if flagRegioMat[ir, ic] > 0:
# indexUsed = np.where(val==potency[ir,ic])[0];
alpha[ir, ic, PREC] = alphaTmp
omega[ir, ic, PREC] = omegaTmp
del (PrecPatch, IndicEq, indexUsed, potency)
#rescale to initial spatial resolution, through nearest interpolation
#initialize variable
omegaFinal = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5))
alphaFinal = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5))
[i, j] = np.where(np.isfinite(omega[:, :, 0]))
for p in range(0, nPrec):
omegaFinal[:, :, p] = omega[i[0], j[0], p]
alphaFinal[:, :, p] = alpha[i[0], j[0], p]
#store final results
conf.omegaFinalStep1_alldom = omegaFinal
conf.alphaFinalStep1_alldom = alphaFinal
conf.ci2Step1 = ci2
conf.CovB2Step1 = CovB2
#######################################################
#######################################################
#START NOW THE SECOND STEP OF THE COMPUTATION OF OMEGA
#######################################################
#######################################################
# convert from 28 to 7 km
Prec = f7.from7to28(conf.Prec)
ny = int(conf.ny / 4)
nx = int(conf.nx / 4)
rad = conf.radStep2
nPrec = conf.nPrec
rf = conf.rf1
flagRegioMat = np.copy(conf.flagRegioMat)
# Y, X = np.mgrid[-rad:rad + 1:1, -rad:rad + 1:1]; # np.meshgrid(r_[-rad:1:rad], r_[-rad:1:rad]);
# F = 1 / ((1 + (X ** 2 + Y ** 2) ** (0.5)));
# pad Prec with zeros around initial matrix, to perform matrix products later on
Prec2 = np.zeros(
(ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3]))
Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :]
Prec = Prec2
# convert from 28 to 7 km
Indic = f7.from7to28(conf.Indic)
flagRegioMat = f7.from7to28(flagRegioMat)
af = conf.alphaFinalStep1_alldom
of = conf.omegaFinalStep1_alldom
ICvecalpha = [
af[np.isfinite(af)][0], af[np.isfinite(af)][1], af[np.isfinite(af)][2],
af[np.isfinite(af)][3], af[np.isfinite(af)][4]
]
ICvecomega = [
of[np.isfinite(of)][0], of[np.isfinite(of)][1], of[np.isfinite(of)][2],
of[np.isfinite(of)][3], of[np.isfinite(of)][4]
]
# initialize variables
omega = np.full([ny, nx, nPrec], 1.5)
# omega = np.full([ny, nx, nPrec], 2);
# for i in range(0,5):
# omega[:,:,i] = of[np.isfinite(of)][i]
ICvecalpha = [1, 1, 1, 1, 1]
ICvecomega = [1.5, 1.5, 1.5, 1.5, 1.5]
# loop over precursors
for precursor in range(0, nPrec):
PREC = precursor
Ide = IdeVec[precursor]
# intialize variables
print('precursor: ' + str(PREC))
# icel=1
# flagIC = 0
t1 = time.time()
#Ide or conf.Ide ???
argslist = (zip(
np.where(flagRegioMat > 0)[0],
np.where(flagRegioMat > 0)[1], repeat(Ide), repeat(rf), repeat(nx),
repeat(ny), repeat(Prec), repeat(PREC), repeat(Indic),
repeat(ICvecalpha), repeat(ICvecomega), repeat(rad),
repeat(conf.ratioPoly), repeat(lat)))
pool = mp.Pool() # by default use available corse
print('***** Using parallel computing with ' + str(mp.cpu_count()) +
' cores *****')
result = pool.starmap_async(computeOutput, argslist)
pool.close()
pool.join()
print(str(time.time() - t1))
res = np.vstack(result.get()) # result as nparray
print(res)
alpha[np.where(flagRegioMat > 0)[0],
np.where(flagRegioMat > 0)[1], PREC] = res[:, 0]
omega[np.where(flagRegioMat > 0)[0],
np.where(flagRegioMat > 0)[1], PREC] = res[:, 1]
conf.omegaFinalStep1_notFiltered = omega
omegaFinal2 = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5))
#from aggregated to initial resolution
for i in range(0, nPrec):
for irAgg in range(0, ny):
for icAgg in range(0, nx):
omegaFinal2[irAgg * 4:irAgg * 4 + 4, icAgg * 4:icAgg * 4 + 4,
i] = omega[irAgg, icAgg, i]
print('precursor interpolated: ' + str(i))
#initialize matrix and if needed apply gaussian filter
omegaFinal = np.zeros_like(omegaFinal2)
for i in range(0, 5):
tmp = omegaFinal2[:, :, i]
if conf.gf == 1: # if gaussian filter has to be applied
tmp = gf.gaussian_filter(tmp, sigma=10)
omegaFinal[:, :, i] = tmp
#rount results and save it to final matrix
omegaFinal = np.round(omegaFinal, 1)
conf.omegaFinalStep1 = omegaFinal
conf.ci2Step1 = []
conf.CovB2Step1 = []
def step1_omegaOptimization(conf):
#convert from 28 to 7 km
Prec = f7.from7to28(conf.Prec);
ny = int(conf.ny/4);
nx = int(conf.nx/4);
rad = conf.radStep1;
nPrec = 5#len(conf.vec3[conf.POLLSEL])#conf.nPrec;
rf = conf.rf1
flagRegioMat = np.copy(conf.flagRegioMat);
#pad Prec with zeros around initial matrix, to perform matrix products later on
Prec2 = np.zeros((ny+rad*2,nx+rad*2,Prec.shape[2],Prec.shape[3]));
Prec2[rad:-rad,rad:-rad,:,:] = Prec[:,:,:,:];
Prec=Prec2;
#convert from 28 to 7 km
Indic = f7.from7to28(conf.Indic);
flagRegioMat = f7.from7to28(flagRegioMat);
lat = f7.from7to28(conf.y);
# flagPerNoxPP??m = f7.from7to28(flagPerNoxPPm);
#initialize variables
omega = np.full([ny,nx,nPrec],1.5);
alpha = np.full([ny,nx,nPrec],np.nan);
ci2 = np.empty((nPrec), dtype=object);
CovB2 = np.empty((nPrec), dtype=object);
# alphaTmp = np.zeros((categories.size));
# omegaTmp = np.zeros((categories.size));
#define training scenarios; note scenarios number is +1 if checking DoE...as in line 74 it is -1
if conf.domain == 'emep10km':
if conf.aqi == 'SURF_ug_PM25_rh50-Yea':
IdeVec = (np.array([1, 1]),np.array([1, 2]),np.array([1, 3]),np.array([1, 5]),np.array([1, 6]));
elif conf.aqi == 'SURF_ug_PM10_rh50-Yea':
IdeVec = (np.array([1, 1]),np.array([1, 2]),np.array([1, 3]),np.array([1, 4]),np.array([1, 6]));
elif conf.domain == 'ineris7km':
IdeVec = (np.array([1, 2]),np.array([1, 3]),np.array([1, 4]),np.array([1, 5]),np.array([1, 6]));
elif (conf.domain == 'emepV433_camsV221') | (conf.domain == 'edgar2015') | (conf.domain == 'emepV434_camsV42'):
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]), np.array([1, 4]), np.array([1, 5]));
#loop over precursors
for precursor in range(0, 5):
PREC = precursor;
Ide = IdeVec[precursor];
# icel = 0;
if (PREC == 2 or PREC== 3):
bnds = ((0, 1), (1.5, 2.5))
else:
bnds = ((0, 1), (1.5, 2.5))
#intialize variables
# numcells = nx*ny
# numcells = np.sum(flagRegioMat>0) # create empty matrix only for really needed points
# PrecPatch = np.zeros((numcells,(rad*2+1)**2));
# IndicEq = np.zeros((numcells,1));
# latVec = np.zeros((numcells,1));
print('precursor: '+str(PREC));
for ic in range(0, nx):
print(PREC, ic);
for ir in range(0, ny):
if flagRegioMat[ir,ic]>0:
#variable to store which group ot be considered
# indexUsed[icel] = np.where(val==potency[ir,ic]);
#create data for omega calculation
nSc = Ide.shape[0]-1;# size(Ide,2)-1
tmpPrec = ep.EquaPrec(ic,ir,rf,nx,ny,nSc,Prec.shape[3],Prec[:,:,Ide[1],PREC],rad); # patches
tmpInde = ei.EquaIndic(ic,ir,rf,nx,ny,nSc,Indic[:,:,Ide[1]]); # indicator
#store data for omega calculation
# PrecPatch[icel,:] = tmpPrec; #np.squeeze(tmpPrec)
# IndicEq[icel] = tmpInde;
latVec = lat[ir,ic]
# icel = icel+1;
# remInd = (tmpInde>0).flatten()
i=1
x0 = [1, 2];
# print(remInd)
# inp1 = tmpPrec[remInd]#[ind,:];
# inp2 = tmpInde[remInd]#[ind];
inp1 = tmpPrec#[ind,:];
inp2 = tmpInde#[ind];
# mdl = minimize(iop, x0, args=(inp1, inp2, rad, latVec, conf.ratioPoly), bounds=bnds, method='BFGS', options=opts) # L-BFGS-B, TNC
opts = {'disp': False}
# mdl = minimize(iop, x0, args=(inp1, inp2, rad, latVec, conf.ratioPoly), method='BFGS', options=opts) # L-BFGS-B, TNC
# print(mdl.x)
mdl = minimize(iop, x0, args=(inp1, inp2, rad, latVec, conf.ratioPoly), bounds=bnds, method='L-BFGS-B', options=opts) # L-BFGS-B, TNC
# print(mdl.x)
# mdl = minimize(iop, x0, args=(inp1, inp2, rad, latVec, conf.ratioPoly), bounds=bnds, method='SLSQP', options=opts) # L-BFGS-B, TNC
# print(mdl.x)
alpha[ir,ic,PREC] = mdl.x[0];
omega[ir,ic,PREC] = mdl.x[1];
#rescale to initial spatial resolution, through nearest interpolation
#initialize variable
conf.omegaFinalStep1_notFiltered = omega
omegaFinal2 = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5));
for i in range(0, nPrec):
for irAgg in range(0, ny):
for icAgg in range(0, nx):
omegaFinal2[irAgg * 4:irAgg * 4 + 4, icAgg * 4:icAgg * 4 + 4, i] = omega[irAgg, icAgg, i]
print('precursor interpolated: ' + str(i));
# omegaFinal = omegaFinal2
omegaFinal = np.zeros_like(omegaFinal2)
for i in range(0, 5):
tmp = omegaFinal2[:, :, i]
tmp = gf.gaussian_filter(tmp, sigma=5)
omegaFinal[:, :, i] = tmp
omegaFinal = np.round(omegaFinal, 1)
# omegaFinal = q.quant(omegaFinal, 0.2) # discretize
#keep only results on the mask
omegaFinal[conf.flagRegioMat==0] =np.nan
conf.omegaFinalStep1 = omegaFinal;
conf.ci2Step1 = [];
conf.CovB2Step1 = [];
def step1_omegaOptPerPoll_7km(conf, idxX, idxY):
conf.scaler = {}
prctileVec1 = np.array([100, 100, 100, 100, 100])
# prctileVec2=np.array([70, 70, 70, 70, 70]);
categories = np.array([1])
#convert from 28 to 7 km
Prec = (conf.Prec)
ny = int(conf.ny)
nx = int(conf.nx)
rad = conf.radStep1
nPrec = len(conf.vec3[conf.POLLSEL]) #conf.nPrec;
rf = conf.rf
flagRegioMat = np.copy(conf.flagRegioMat_FUL)
#pad Prec with zeros around initial matrix, to perform matrix products later on
Prec2 = np.zeros(
(ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3]))
Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :]
Prec = Prec2
#convert from 28 to 7 km
Indic = (conf.Indic)
# flagRegioMatFull = (flagRegioMat);
# flagPerNoxPPm = (flagPerNoxPPm);
#initialize variables
omega = np.full([ny, nx, nPrec], np.nan)
alpha = np.full([ny, nx, nPrec], np.nan)
ci2 = np.empty((nPrec), dtype=object)
CovB2 = np.empty((nPrec), dtype=object)
# alphaTmp = np.zeros((categories.size));
# omegaTmp = np.zeros((categories.size));
#define training scenarios; note scenarios number is +1 if checking DoE...as in line 74 it is -1
if conf.domain == 'emep10km':
if conf.aqi == 'SURF_ug_PM25_rh50-Yea':
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 5]), np.array([1, 6]))
elif conf.aqi == 'SURF_ug_PM10_rh50-Yea':
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 4]), np.array([1, 6]))
elif conf.domain == 'ineris7km':
IdeVec = (np.array([1, 2]), np.array([1, 3]), np.array([1, 4]),
np.array([1, 5]), np.array([1, 6]))
#loop over precursors
for precursor in range(0, nPrec):
maxValLoop = 11
for numRings in range(3, maxValLoop):
print('testing ' + str(numRings) + ' rings of cells')
flagPerNoxVocPpmSo2 = np.zeros_like(conf.flagRegioMat)
flagPerNoxVocPpmSo2[idxY - numRings:idxY + numRings + 1,
idxX - numRings:idxX + numRings + 1] = 1
PREC = precursor
Ide = IdeVec[precursor]
icel = 0
#################################################################
# PART TO BE MODIFIED FOR TESTING LOCALIZATION
#flagRegioMat = flagRegioMatFull
# if (precursor==0) | (precursor==1) | (precursor==3) | (precursor==4):
flagRegioMat = flagPerNoxVocPpmSo2
# else:
# flagRegioMat = flagPerNH3
#################################################################
#intialize variables
numcells = nx * ny
numcells = np.sum(
flagRegioMat >
0) # create empty matrix only for really needed points
PrecPatch = np.zeros((numcells, (rad * 2 + 1)**2))
IndicEq = np.zeros((numcells, 1))
indexUsed = np.full((numcells, 1), np.nan)
#np.zeros((nx*ny,1));
potency = np.full((numcells), np.nan)
#np.zeros((ny,nx));
for ic in range(0, nx):
#print(PREC, ic);
for ir in range(0, ny):
if flagRegioMat[ir, ic] > 0:
#variable to store which group ot be considered
# indexUsed[icel] = np.where(val==potency[ir,ic]);
#create data for omega calculation
nSc = Ide.shape[0] - 1
# size(Ide,2)-1
tmpPrec = ep.EquaPrec(ic, ir, rf, nx, ny, nSc,
Prec.shape[3], Prec[:, :, Ide[1],
PREC], rad)
# patches
tmpInde = ei.EquaIndic(ic, ir, rf, nx, ny, nSc,
Indic[:, :, Ide[1]])
# indicator
#store data for omega calculation
PrecPatch[icel, :] = tmpPrec
#np.squeeze(tmpPrec)
IndicEq[icel] = tmpInde
icel = icel + 1
# indexUsedLin = np.reshape(indexUsed, -1, order='F');
#compute omega for each group of cells, given precursor p
# for i in range(val.size):
remInd = (IndicEq > 0).flatten()
i = 1
x0 = [1, 2]
# ind = np.where(indexUsedLin==i)[0];
inp1 = PrecPatch[remInd] #[ind,:];
inp2 = IndicEq[remInd] #[ind];
# iop = lambda inp1,beta1,beta2: inv.InvDistN_opt_prec([beta1,beta2],inp1,rad);
#
# [mdl,r,J,CovB] = nlin.nlinfit(iop,inp1,inp2.ravel(),x0);
bnds = ((-2, 2), (0, 4))
# opt = {'ftol':1e-12, 'gtol':1e-12}
opt = {
'disp': False,
'xtol': 0.0001,
'ftol': 0.0001,
'maxiter': 2000,
'maxfev': 2000
}
# mdl = minimize(iop, x0, args=(inp1, inp2, rad), bounds=bnds, method='L-BFGS-B')#, options = opt) # L-BFGS-B, TNC
mdl = minimize(iop,
x0,
args=(inp1, inp2, rad),
bounds=bnds,
method='L-BFGS-B',
options=opt) # L-BFGS-B, TNC
if (mdl.x[1] < 0.2) | (mdl.x[1] > 3.8):
if numRings == maxValLoop - 1:
mdl.x[1] = np.unique(
conf.omegaFinalStep1_28km[idxY - 1:idxY + 1,
idxX - 1:idxX + 1,
precursor])[0]
else:
continue
# scaler = preprocessing.MinMaxScaler().fit(inp1)
# inp1 = scaler.transform(inp1)
# conf.scaler[precursor] = scaler
# mdl = minimize(iop, x0, args=(inp1, inp2, rad), method='Powell', options=opt) # L-BFGS-B, TNC
# print('prec' + str(precursor))
print('Precursor: ' + str(PREC) + '. Optimization success: ' +
str(mdl.success) + '. Omega value: ' + str(mdl.x[1]))
print('success with ' + str(numRings) + ' rings!')
break
# ci2[PREC] = nlpa.nlparci(r,J);
# CovB2[PREC] = CovB;
alphaTmp = mdl.x[0]
omegaTmp = mdl.x[1]
#repeat result for each belonging to a given group
for ic in range(0, nx):
for ir in range(0, ny):
if flagRegioMat[ir, ic] > 0:
# indexUsed = np.where(val==potency[ir,ic])[0];
alpha[ir, ic, PREC] = alphaTmp
omega[ir, ic, PREC] = omegaTmp
del (PrecPatch, IndicEq, indexUsed, potency)
#rescale to initial spatial resolution, through nearest interpolation
#initialize variable
omegaFinal = omega #np.zeros((conf.Prec.shape[0],conf.Prec.shape[1],5));
# #loop on precursors
# for i in range(0, nPrec):
# #define interpolator object
# xgv = np.arange(1., conf.Prec.shape[0]/4+1);
# ygv = np.arange(1., conf.Prec.shape[1]/4+1);
# F=interpol.RegularGridInterpolator((xgv, ygv), omega[:,:,i],method='nearest',bounds_error=False, fill_value=None);
#
# #interpolate
# Xq = np.arange(1., conf.Prec.shape[0]/4+1, 1/4);
# Yq = np.arange(1., conf.Prec.shape[1]/4+1, 1/4);
# [Y2,X2] = np.meshgrid(Yq, Xq);
# pts=((X2.flatten(),Y2.flatten()))
# omegaFinal[:,:,i] = F(pts).reshape(conf.Prec.shape[0],conf.Prec.shape[1])
# print('precursor interpolated: '+str(i));
#
#store final results
conf.omegaFinalStep1 = omegaFinal
conf.ci2Step1 = ci2
conf.CovB2Step1 = CovB2
def step1_omegaOptimization(conf):
prctileVec1 = np.array([100, 100, 100, 100, 100])
# prctileVec2=np.array([70, 70, 70, 70, 70]);
categories = np.array([1])
#convert from 28 to 7 km
Prec = f7.from7to28(conf.Prec)
ny = int(conf.ny / 4)
nx = int(conf.nx / 4)
rad = conf.radStep1
nPrec = len(conf.vec3[conf.POLLSEL]) #conf.nPrec;
rf = 0
flagRegioMat = np.copy(conf.flagRegioMat)
#pad Prec with zeros around initial matrix, to perform matrix products later on
Prec2 = np.zeros(
(ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3]))
Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :]
Prec = Prec2
#convert from 28 to 7 km
Indic = f7.from7to28(conf.Indic)
flagRegioMat = f7.from7to28(flagRegioMat)
lat = f7.from7to28(conf.y)
# flagPerNoxPP??m = f7.from7to28(flagPerNoxPPm);
#initialize variables
omega = np.full([ny, nx, nPrec], np.nan)
alpha = np.full([ny, nx, nPrec], np.nan)
ci2 = np.empty((nPrec), dtype=object)
CovB2 = np.empty((nPrec), dtype=object)
# alphaTmp = np.zeros((categories.size));
# omegaTmp = np.zeros((categories.size));
#define training scenarios; note scenarios number is +1 if checking DoE...as in line 74 it is -1
if conf.domain == 'emep10km':
if conf.aqi == 'SURF_ug_PM25_rh50-Yea':
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 5]), np.array([1, 6]))
elif conf.aqi == 'SURF_ug_PM10_rh50-Yea':
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 4]), np.array([1, 6]))
elif conf.domain == 'ineris7km':
IdeVec = (np.array([1, 2]), np.array([1, 3]), np.array([1, 4]),
np.array([1, 5]), np.array([1, 6]))
elif (conf.domain == 'emepV433_camsV221') | (
conf.domain == 'edgar2015') | (conf.domain == 'emepV434_camsV42'):
IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]),
np.array([1, 4]), np.array([1, 5]))
#loop over precursors
for precursor in range(0, nPrec):
PREC = precursor
Ide = IdeVec[precursor]
icel = 0
#intialize variables
numcells = nx * ny
numcells = np.sum(
flagRegioMat >
0) # create empty matrix only for really needed points
PrecPatch = np.zeros((numcells, (rad * 2 + 1)**2))
IndicEq = np.zeros((numcells, 1))
latVec = np.zeros((numcells, 1))
indexUsed = np.full((numcells, 1), np.nan)
#np.zeros((nx*ny,1));
potency = np.full((numcells), np.nan)
#np.zeros((ny,nx));
print('precursor: ' + str(PREC))
for ic in range(0, nx):
#print(PREC, ic);
for ir in range(0, ny):
if flagRegioMat[ir, ic] > 0:
#variable to store which group ot be considered
# indexUsed[icel] = np.where(val==potency[ir,ic]);
#create data for omega calculation
nSc = Ide.shape[0] - 1
# size(Ide,2)-1
tmpPrec = ep.EquaPrec(ic, ir, rf, nx, ny, nSc,
Prec.shape[3], Prec[:, :, Ide[1],
PREC], rad)
# patches
tmpInde = ei.EquaIndic(ic, ir, rf, nx, ny, nSc,
Indic[:, :, Ide[1]])
# indicator
#store data for omega calculation
PrecPatch[icel, :] = tmpPrec
#np.squeeze(tmpPrec)
IndicEq[icel] = tmpInde
latVec[icel] = lat[ir, ic]
icel = icel + 1
# indexUsedLin = np.reshape(indexUsed, -1, order='F');
#compute omega for each group of cells, given precursor p
# for i in range(val.size):
remInd = (IndicEq > 0).flatten()
i = 1
x0 = [1, 2]
# ind = np.where(indexUsedLin==i)[0];
inp1 = PrecPatch[remInd] #[ind,:];
inp2 = IndicEq[remInd] #[ind];
latVecFilt = latVec[remInd]
#rescaling input between min and max
# scaler = MinMaxScaler(feature_range=(0, 1))
# scaler.fit(inp1)
# inp1 = scaler.transform(inp1)
# iop = lambda inp1,beta1,beta2: inv.InvDistN_opt_prec([beta1,beta2],inp1,rad);
# [mdl,r,J,CovB] = nlin.nlinfit(iop,inp1,inp2.ravel(),x0);
bnds = ((0, 2), (0.1, 2.9))
opts = {'disp': False, 'ftol': 10e-6}
mdl = minimize(iop,
x0,
args=(inp1, inp2, rad, latVecFilt, conf.ratioPoly),
bounds=bnds,
method='L-BFGS-B',
options=opts) # L-BFGS-B, TNC
# ?import scipy.optimize.dif as bs
#mdl = bs(iop, x0, args=(inp1, inp2, rad, latVecFilt, conf.ratioPoly), bounds=bnds, method='L-BFGS-B', options=opts) # L-BFGS-B, TNC
# print('prec' + str(precursor))
# print(mdl)
# ci2[PREC] = nlpa.nlparci(r,J);
# CovB2[PREC] = CovB;
alphaTmp = mdl.x[0]
omegaTmp = mdl.x[1]
#print(alphaTmp)
#repeat result for each belonging to a given group
for ic in range(0, nx):
for ir in range(0, ny):
if flagRegioMat[ir, ic] > 0:
# indexUsed = np.where(val==potency[ir,ic])[0];
alpha[ir, ic, PREC] = alphaTmp
omega[ir, ic, PREC] = omegaTmp
del (PrecPatch, IndicEq, indexUsed, potency)
#rescale to initial spatial resolution, through nearest interpolation
#initialize variable
omegaFinal = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5))
for i in range(0, 5):
omegaFinal[:, :, i] = np.unique(omega[:, :, i])[0]
#loop on precursors
# for i in range(0, nPrec):
# #define interpolator object
# xgv = np.arange(1., conf.Prec.shape[0]/4+1);
# ygv = np.arange(1., conf.Prec.shape[1]/4+1);
# F=interpol.RegularGridInterpolator((xgv, ygv), omega[:,:,i],method='nearest',bounds_error=False, fill_value=None);
#
# #interpolate
# Xq = np.arange(1., conf.Prec.shape[0]/4+1, 1/4);
# Yq = np.arange(1., conf.Prec.shape[1]/4+1, 1/4);
# [Y2,X2] = np.meshgrid(Yq, Xq);
# pts=((X2.flatten(),Y2.flatten()))
# omegaFinal[:,:,i] = F(pts).reshape(conf.Prec.shape[0],conf.Prec.shape[1])
# print('precursor interpolated: '+str(i));
#store final results
# replacingVal = np.unique(omegaFinal[:,:,whichPollToUpdate][~np.isnan(omegaFinal[:,:,whichPollToUpdate])])
# conf.omegaFinalStep1[:,:,whichPollToUpdate] = replacingVal#omegaFinal[:,:,whichPollToUpdate];
conf.omegaFinalStep1 = omegaFinal
# conf.omegaFinalStep1_28km = omegaFinal
conf.ci2Step1 = ci2
conf.CovB2Step1 = CovB2