def step1_potency(conf): prctileVec1 = np.array([35, 35, 35, 35, 35]) prctileVec2 = np.array([70, 70, 70, 70, 70]) categories = np.array([1, 2, 3]) #convert from 28 to 7 km Prec = f7.from7to28(conf.Prec) ny = int(conf.ny / 4) nx = int(conf.nx / 4) rad = conf.radStep1 nPrec = conf.nPrec rf = conf.rf flagRegioMat = np.copy(conf.flagRegioMat) #pad Prec with zeros around initial matrix, to perform matrix products later on Prec2 = np.zeros( (ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3])) Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :] Prec = Prec2 #convert from 28 to 7 km Indic = f7.from7to28(conf.Indic) flagRegioMat = f7.from7to28(flagRegioMat) #initialize variables omega = np.full([ny, nx, nPrec], np.nan) alpha = np.full([ny, nx, nPrec], np.nan) ci2 = np.empty((categories.size, nPrec), dtype=object) CovB2 = np.empty((categories.size, nPrec), dtype=object) alphaTmp = np.zeros((categories.size)) omegaTmp = np.zeros((categories.size)) #define training scenarios; note scenarios number is +1 if checking DoE...as in line 74 it is -1 if conf.domain == 'emep10km': if conf.aqi == 'SURF_ug_PM25_rh50-Yea': IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]), np.array([1, 5]), np.array([1, 6])) elif conf.aqi == 'SURF_ug_PM10_rh50-Yea': IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]), np.array([1, 4]), np.array([1, 6])) elif conf.domain == 'ineris7km': IdeVec = (np.array([1, 8]), np.array([1, 9]), np.array([1, 10]), np.array([1, 11]), np.array([1, 12])) #loop over precursors for precursor in range(0, nPrec): PREC = precursor Ide = IdeVec[precursor] icel = 0 #intialize variables PrecPatch = np.zeros((nx * ny, (rad * 2 + 1)**2)) IndicEq = np.zeros((nx * ny, 1)) indexUsed = np.full((nx * ny, 1), np.nan) #np.zeros((nx*ny,1)); potency = np.full((ny, nx), np.nan) #np.zeros((ny,nx)); print('precursor: ' + str(PREC)) #loop over cells to create groups for ic in range(0, nx): #print(PREC, ic); for ir in range(0, ny): if flagRegioMat[ir, ic] > 0: #create data for omega calculation nSc = Ide.shape[0] - 1 # size(Ide,2)-1 tmpPrec = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide[1], PREC], rad) # patches tmpInde = ei.EquaIndic(ic, ir, rf, nx, ny, nSc, Indic[:, :, Ide[1]]) # indicator x0 = np.array([1, 2]) [inp2_aggemi] = inv.InvDistN_opt_prec(x0, tmpPrec, rad) #store data for omega calculation potency[ir, ic] = tmpInde / inp2_aggemi prc1 = np.percentile(potency[np.isfinite(potency)], prctileVec1[precursor]) prc9 = np.percentile(potency[np.isfinite(potency)], prctileVec2[precursor]) speed = potency.copy() speed[np.isnan(speed)] = 0 potency[speed < prc1] = 1 potency[(speed >= prc1) & (speed < prc9)] = 2 potency[speed >= prc9] = 3 val = categories for ic in range(0, nx): #print(PREC, ic); for ir in range(0, ny): if flagRegioMat[ir, ic] > 0: #variable to store which group ot be considered indexUsed[icel] = np.where(val == potency[ir, ic]) #create data for omega calculation nSc = Ide.shape[0] - 1 # size(Ide,2)-1 tmpPrec = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide[1], PREC], rad) # patches tmpInde = ei.EquaIndic(ic, ir, rf, nx, ny, nSc, Indic[:, :, Ide[1]]) # indicator #store data for omega calculation PrecPatch[icel, :] = tmpPrec #np.squeeze(tmpPrec) IndicEq[icel] = tmpInde icel = icel + 1 indexUsedLin = np.reshape(indexUsed, -1, order='F') #compute omega for each group of cells, given precursor p for i in range(val.size): x0 = [1, 2] ind = np.where(indexUsedLin == i)[0] inp1 = PrecPatch[ind, :] inp2 = IndicEq[ind] iop = lambda inp1, beta1, beta2: inv.InvDistN_opt_prec( [beta1, beta2], inp1, rad) [mdl, r, J, CovB] = nlin.nlinfit(iop, inp1, inp2.ravel(), x0) ci2[i, PREC] = nlpa.nlparci(r, J) CovB2[i, PREC] = CovB alphaTmp[i] = mdl[0] omegaTmp[i] = mdl[1] #repeat result for each belonging to a given group for ic in range(0, nx): for ir in range(0, ny): if flagRegioMat[ir, ic] > 0: indexUsed = np.where(val == potency[ir, ic])[0] alpha[ir, ic, PREC] = alphaTmp[indexUsed] omega[ir, ic, PREC] = omegaTmp[indexUsed] del (PrecPatch, IndicEq, indexUsed, potency, speed) #rescale to initial spatial resolution, through nearest interpolation #initialize variable omegaFinal = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5)) #loop on precursors for i in range(0, nPrec): #define interpolator object xgv = np.arange(1., conf.Prec.shape[0] / 4 + 1) ygv = np.arange(1., conf.Prec.shape[1] / 4 + 1) F = interpol.RegularGridInterpolator((xgv, ygv), omega[:, :, i], method='nearest', bounds_error=False, fill_value=None) #interpolate Xq = np.arange(1., conf.Prec.shape[0] / 4 + 1, 1 / 4) Yq = np.arange(1., conf.Prec.shape[1] / 4 + 1, 1 / 4) [Y2, X2] = np.meshgrid(Yq, Xq) pts = ((X2.flatten(), Y2.flatten())) omegaFinal[:, :, i] = F(pts).reshape(conf.Prec.shape[0], conf.Prec.shape[1]) print('precursor interpolated: ' + str(i)) #store final results conf.omegaFinalStep1 = omegaFinal conf.ci2Step1 = ci2 conf.CovB2Step1 = CovB2
def step1_omegaOptimization(conf): #convert from 28 to 7 km # Prec = conf.Prec; # ny = conf.ny; # nx = conf.nx; Prec = f7.from7to28(conf.Prec) ny = int(conf.ny / 4) nx = int(conf.nx / 4) rad = conf.radStep1 nPrec = conf.nPrec rf = conf.rf1 flagRegioMat = np.copy(conf.flagRegioMat) #pad Prec with zeros around initial matrix, to perform matrix products later on Prec2 = np.zeros( (ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3])) Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :] Prec = Prec2 #convert from 28 to 7 km # Indic = conf.Indic; # flagRegioMat = flagRegioMat; Indic = f7.from7to28(conf.Indic) flagRegioMat = f7.from7to28(flagRegioMat) lat = f7.from7to28(conf.y) #initialize variables omega = np.full([ny, nx, nPrec], np.nan) alpha = np.full([ny, nx, nPrec], np.nan) #precomputed vars Y, X = np.mgrid[-rad:rad + 1:1, -rad:rad + 1:1] distanceMat = 1 / ((1 + (X**2 + Y**2)**(0.5))) distanceMat = distanceMat.flatten() #define training scenarios; note scenarios number is +1 if checking DoE...as in line 74 it is -1 if conf.domain == 'emep10km': if conf.aqi == 'SURF_ug_PM25_rh50-Yea': IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]), np.array([1, 5]), np.array([1, 6])) elif conf.aqi == 'SURF_ug_PM10_rh50-Yea': IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]), np.array([1, 4]), np.array([1, 6])) elif conf.domain == 'ineris7km': IdeVec = (np.array([1, 2]), np.array([1, 3]), np.array([1, 4]), np.array([1, 5]), np.array([1, 6])) elif (conf.domain == 'emepV433_camsV221') | ( conf.domain == 'edgar2015') | (conf.domain == 'emepV434_camsV42'): IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]), np.array([1, 4]), np.array([1, 5])) for precursor in range(0, nPrec): Ide = conf.Ide PREC = precursor Ide = IdeVec[precursor] for ic in range(0, nx): print(ic, nx) for ir in range(0, ny): if flagRegioMat[ir, ic] > 0: # ratio = np.polyval(conf.ratioPoly, conf.y[ir, ic]) ratio = np.polyval(conf.ratioPoly, lat[ir, ic]) Y, X = np.mgrid[-rad:rad + 1:1, -rad:rad + 1:1] distanceMat = 1 / ((1 + ((X / ratio)**2 + Y**2)**0.5)) distanceMat = distanceMat.flatten() # ratio = np.polyval(conf.ratioPoly, lat[ir, ic]) # Y, X = np.mgrid[-rad:rad + 1:1, -rad:rad + 1:1]; # F = 1 / ((1 + ((X / ratio) ** 2 + Y ** 2) ** 0.5)); # # Y, X = np.mgrid[-rad:rad+1:1, -rad:rad+1:1]; # distanceMat = 1 / ( (1 + (X**2 + Y**2)**(0.5))) # distanceMat = distanceMat.flatten() x0 = [1, 1, 1, 1, 1, 1.5, 1.5, 1.5, 1.5, 1.5] nSc = Ide.shape[0] #-1;# size(Ide,2)-1 # t1=t.time(); inp0 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide, 0], rad) # patches inp1 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide, 1], rad) # patches inp2 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide, 2], rad) # patches inp3 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide, 3], rad) # patches inp4 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide, 4], rad) # patches # print("1: %s" % (t.time() - t1)); # t1=t.time(); out0 = ei.EquaIndic(ic, ir, rf, nx, ny, nSc, Indic[:, :, Ide]) # indicator out0 = out0.flatten() # print("2: %s" % (t.time() - t1)); #remove 0 elements idxToRemove = np.where(out0 == 0) out0 = np.delete(out0, idxToRemove) inp0 = np.delete(inp0, idxToRemove, axis=0) inp1 = np.delete(inp1, idxToRemove, axis=0) inp2 = np.delete(inp2, idxToRemove, axis=0) inp3 = np.delete(inp3, idxToRemove, axis=0) inp4 = np.delete(inp4, idxToRemove, axis=0) bnds1 = tuple((0, 5) for n in range(0, 5)) bnds2 = tuple((1, 3) for n in range(0, 5)) bnds = bnds1 + bnds2 #DO NOT FIND OPTIMAL SOLUTION # t1=t.time(); # mdl = minimize(iop, x0, args=(inp0, inp1, inp2, inp3, inp4, out0, distanceMat), method='Nelder-Mead', options={'maxiter':500}) # print (t.time()-t1) # #OK # t1=t.time(); # mdl = minimize(iop, x0, args=(inp0, inp1, inp2, inp3, inp4, out0, distanceMat), bounds=bnds, method='L-BFGS-B', options={'maxiter':500}) # print (t.time()-t1) # # t1=t.time(); # mdl = minimize(iop, x0, args=(inp0, inp1, inp2, inp3, inp4, out0, distanceMat), method='COBYLA', options={'maxiter':500}) # (t.time()-t1) # # t1=t.time(); mdl = minimize(iop, x0, args=(inp0, inp1, inp2, inp3, inp4, out0, distanceMat), bounds=bnds, method='SLSQP', options={'maxiter': 500}) # print(mdl) # print("4: %s" % (t.time() - t1)); # print(mdl.x[5:]) omega[ir, ic, :] = mdl.x[5:] alpha[ir, ic, :] = mdl.x[0:5] conf.omegaFinalStep1_notFiltered = omega omegaFinal2 = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5)) for i in range(0, nPrec): for irAgg in range(0, ny): for icAgg in range(0, nx): omegaFinal2[irAgg * 4:irAgg * 4 + 4, icAgg * 4:icAgg * 4 + 4, i] = omega[irAgg, icAgg, i] print('precursor interpolated: ' + str(i)) # omegaFinal = omegaFinal2 omegaFinal = np.zeros_like(omegaFinal2) for i in range(0, 5): tmp = omegaFinal2[:, :, i] tmp = gf.gaussian_filter(tmp, sigma=.5) omegaFinal[:, :, i] = tmp omegaFinal = np.round(omegaFinal, 1) # omegaFinal = q.quant(omegaFinal, 0.2) # discretize #keep only results on the mask omegaFinal[conf.flagRegioMat == 0] = np.nan conf.omegaFinalStep1 = omegaFinal conf.ci2Step1 = [] conf.CovB2Step1 = []
def step1_omegaOptSliding_aggRes_perPoll(conf): prctileVec1 = np.array([100, 100, 100, 100, 100]) # prctileVec2=np.array([70, 70, 70, 70, 70]); categories = np.array([1]) #convert from 28 to 7 km Prec = f7.from7to28(conf.Prec) ny = int(conf.ny / 4) nx = int(conf.nx / 4) rad = conf.radStep1 nPrec = len(conf.vec3[conf.POLLSEL]) #conf.nPrec; rf = 0 #conf.rf1; flagRegioMat = np.copy(conf.flagRegioMat) #pad Prec with zeros around initial matrix, to perform matrix products later on Prec2 = np.zeros( (ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3])) Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :] Prec = Prec2 #convert from 28 to 7 km Indic = f7.from7to28(conf.Indic) flagRegioMat = f7.from7to28(flagRegioMat) #initialize variables omega = np.full([ny, nx, nPrec], np.nan) alpha = np.full([ny, nx, nPrec], np.nan) ci2 = np.empty((nPrec), dtype=object) CovB2 = np.empty((nPrec), dtype=object) # alphaTmp = np.zeros((categories.size)); # omegaTmp = np.zeros((categories.size)); # define training scenarios; note scenarios number is +1 if checking DoE...as in line 74 it is -1 if conf.domain == 'emep10km': # if conf.aqi == 'SURF_ug_PM25_rh50-Yea': if 'SURF_ug_PM25_rh50' in conf.aqi: IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]), np.array([1, 5]), np.array([1, 6])) # elif conf.aqi == 'SURF_ug_PM10_rh50-Yea': elif 'SURF_ug_PM10_rh50' in conf.aqi: IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]), np.array([1, 4]), np.array([1, 6])) elif 'SURF_ppb_O3' in conf.aqi: IdeVec = (np.array([1, 1]), np.array([1, 2])) elif 'SURF_ug_NOx' in conf.aqi: IdeVec = (np.array([1, 1]), np.array([1, 2])) elif conf.domain == 'ineris7km': IdeVec = (np.array([1, 2]), np.array([1, 3]), np.array([1, 4]), np.array([1, 5]), np.array([1, 6])) #loop over precursors for precursor in range(0, nPrec): PREC = precursor Ide = IdeVec[precursor] icel = 0 #intialize variables numcells = nx * ny numcells = np.sum( flagRegioMat > 0) # create empty matrix only for really needed points PrecPatch = np.zeros((numcells, (rad * 2 + 1)**2)) IndicEq = np.zeros((numcells, 1)) indexUsed = np.full((numcells, 1), np.nan) #np.zeros((nx*ny,1)); potency = np.full((numcells), np.nan) #np.zeros((ny,nx)); print('precursor: ' + str(PREC)) for ic in range(0, nx): #print(PREC, ic); for ir in range(0, ny): if flagRegioMat[ir, ic] > 0: #variable to store which group ot be considered # indexUsed[icel] = np.where(val==potency[ir,ic]); #create data for omega calculation nSc = Ide.shape[0] - 1 # size(Ide,2)-1 tmpPrec = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide[1], PREC], rad) # patches tmpInde = ei.EquaIndic(ic, ir, rf, nx, ny, nSc, Indic[:, :, Ide[1]]) # indicator #store data for omega calculation PrecPatch[icel, :] = tmpPrec #np.squeeze(tmpPrec) IndicEq[icel] = tmpInde icel = icel + 1 # indexUsedLin = np.reshape(indexUsed, -1, order='F'); #compute omega for each group of cells, given precursor p # for i in range(val.size): remInd = (IndicEq > 0).flatten() i = 1 x0 = [1, 1.5] # ind = np.where(indexUsedLin==i)[0]; inp1 = PrecPatch[remInd] #[ind,:]; inp2 = IndicEq[remInd] #[ind]; iop = lambda inp1, beta1, beta2: inv.InvDistN_opt_prec([beta1, beta2], inp1, rad) [mdl, r, J, CovB] = nlin.nlinfit(iop, inp1, inp2.ravel(), x0) # ci2[PREC] = nlpa.nlparci(r,J); # CovB2[PREC] = CovB; alphaTmp = mdl[0] omegaTmp = mdl[1] #repeat result for each belonging to a given group for ic in range(0, nx): for ir in range(0, ny): if flagRegioMat[ir, ic] > 0: # indexUsed = np.where(val==potency[ir,ic])[0]; alpha[ir, ic, PREC] = alphaTmp omega[ir, ic, PREC] = omegaTmp del (PrecPatch, IndicEq, indexUsed, potency) #rescale to initial spatial resolution, through nearest interpolation #initialize variable omegaFinal = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5)) alphaFinal = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5)) [i, j] = np.where(np.isfinite(omega[:, :, 0])) for p in range(0, nPrec): omegaFinal[:, :, p] = omega[i[0], j[0], p] alphaFinal[:, :, p] = alpha[i[0], j[0], p] #store final results conf.omegaFinalStep1_alldom = omegaFinal conf.alphaFinalStep1_alldom = alphaFinal conf.ci2Step1 = ci2 conf.CovB2Step1 = CovB2 ####################################################### ####################################################### #START NOW THE SECOND STEP OF THE COMPUTATION OF OMEGA ####################################################### ####################################################### # convert from 28 to 7 km Prec = f7.from7to28(conf.Prec) ny = int(conf.ny / 4) nx = int(conf.nx / 4) rad = conf.radStep2 nPrec = conf.nPrec rf = conf.rf1 flagRegioMat = np.copy(conf.flagRegioMat) Y, X = np.mgrid[-rad:rad + 1:1, -rad:rad + 1:1] # np.meshgrid(r_[-rad:1:rad], r_[-rad:1:rad]); F = 1 / ((1 + (X**2 + Y**2)**(0.5))) # pad Prec with zeros around initial matrix, to perform matrix products later on Prec2 = np.zeros( (ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3])) Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :] Prec = Prec2 # convert from 28 to 7 km Indic = f7.from7to28(conf.Indic) flagRegioMat = f7.from7to28(flagRegioMat) af = conf.alphaFinalStep1_alldom of = conf.omegaFinalStep1_alldom ICvecalpha = [ af[np.isfinite(af)][0], af[np.isfinite(af)][1], af[np.isfinite(af)][2], af[np.isfinite(af)][3], af[np.isfinite(af)][4] ] ICvecomega = [ of[np.isfinite(of)][0], of[np.isfinite(of)][1], of[np.isfinite(of)][2], of[np.isfinite(of)][3], of[np.isfinite(of)][4] ] # initialize variables omega = np.full([ny, nx, nPrec], 1.5) # omega = np.full([ny, nx, nPrec], 2); # for i in range(0,5): # omega[:,:,i] = of[np.isfinite(of)][i] # ICvecalpha = [1, 1, 1, 1, 1] # ICvecomega = [1.5, 1.5, 1.5, 2.6, 1.5] # loop over precursors for precursor in range(0, nPrec): PREC = precursor Ide = IdeVec[precursor] # intialize variables print('precursor: ' + str(PREC)) # icel=1 # flagIC = 0 for ic in range(0, nx): print(PREC, ic) for ir in range(0, ny): if flagRegioMat[ir, ic] > 0: nSc = Ide.shape[0] - 1 # size(Ide,2)-1 inp1 = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide[1], PREC], rad) # patches inp2 = ei.EquaIndic(ic, ir, rf, nx, ny, nSc, Indic[:, :, Ide[1]]) # indicator # print(inp1.shape) remInd = (inp2 > 0).flatten() inp1 = inp1[remInd] # [ind,:]; inp2 = inp2[remInd] # [ind]; x0tmp_alpha = ICvecalpha[precursor] x0tmp_omega = ICvecomega[precursor] x0 = [x0tmp_alpha, x0tmp_omega] bndstmp_alpha = (0, 10 ) # x0tmp_alpha - 0.5, x0tmp_alpha + 0.5) bndstmp_omega = (x0tmp_omega - 0.5, x0tmp_omega + 0.5) # bnds = ((0, 2), bndstmp) bnds = (bndstmp_alpha, bndstmp_omega) # a=time.time() opts = {'disp': False, 'ftol': 10e-6} mdl = minimize(iop2, x0, args=(inp1, inp2, rad, F), bounds=bnds, method='L-BFGS-B', options=opts) # L-BFGS-B, COBYLA SLSQP # print(mdl.success) omega[ir, ic, PREC] = mdl.x[1] # mdl[1] # if (mdl.x[1]<1.1) or (mdl.x[1]>1.9) : # mdl.x[1] = ICvec[precursor] # omega[ir, ic, PREC] = mdl.x[1]; # mdl[1] # if (mdl.success == True) and (mdl.x[1] > 1) and (mdl.x[1] < 3) and (mdl.x[1] != x0[1]): # conf.omegaAccuracy[ir,ic,PREC] = 2 # omega[ir, ic, PREC] = mdl.x[1]; # mdl[1]; # # print(mdl) # else: # omega[ir, ic, PREC] = conf.step1_omegaOptPerPoll_allCells[ir, ic , PREC] # omega = np.round(omega,1) conf.omegaFinalStep1_notFiltered = omega omegaFinal2 = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5)) for i in range(0, nPrec): for irAgg in range(0, ny): for icAgg in range(0, nx): omegaFinal2[irAgg * 4:irAgg * 4 + 4, icAgg * 4:icAgg * 4 + 4, i] = omega[irAgg, icAgg, i] print('precursor interpolated: ' + str(i)) # omegaFinal = omegaFinal2 omegaFinal = np.zeros_like(omegaFinal2) for i in range(0, 5): tmp = omegaFinal2[:, :, i] tmp = gf.gaussian_filter(tmp, sigma=10) omegaFinal[:, :, i] = tmp omegaFinal = np.round(omegaFinal, 1) # omegaFinal = q.quant(omegaFinal, 0.2) # discretize conf.omegaFinalStep1 = omegaFinal conf.ci2Step1 = [] conf.CovB2Step1 = []
def step1_omegaOptimization(conf): prctileVec1 = np.array([100, 100, 100, 100, 100]) # prctileVec2=np.array([70, 70, 70, 70, 70]); categories = np.array([1]) #convert from 28 to 7 km Prec = f7.from7to28(conf.Prec) ny = int(conf.ny / 4) nx = int(conf.nx / 4) rad = conf.radStep1 nPrec = len(conf.vec3[conf.POLLSEL]) #conf.nPrec; rf = 0 #conf.rf1; flagRegioMat = np.copy(conf.flagRegioMat) #pad Prec with zeros around initial matrix, to perform matrix products later on Prec2 = np.zeros( (ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3])) Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :] Prec = Prec2 #convert from 28 to 7 km Indic = f7.from7to28(conf.Indic) flagRegioMat = f7.from7to28(flagRegioMat) lat = f7.from7to28(conf.y) #initialize variables omega = np.full([ny, nx, nPrec], np.nan) alpha = np.full([ny, nx, nPrec], np.nan) ci2 = np.empty((nPrec), dtype=object) CovB2 = np.empty((nPrec), dtype=object) # alphaTmp = np.zeros((categories.size)); # omegaTmp = np.zeros((categories.size)); # define training scenarios; note scenarios number is +1 if checking DoE...as in line 74 it is -1 if conf.domain == 'emep10km': # if conf.aqi == 'SURF_ug_PM25_rh50-Yea': if 'SURF_ug_PM25_rh50' in conf.aqi: IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]), np.array([1, 5]), np.array([1, 6])) # elif conf.aqi == 'SURF_ug_PM10_rh50-Yea': elif 'SURF_ug_PM10_rh50' in conf.aqi: IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]), np.array([1, 4]), np.array([1, 6])) elif 'SURF_ppb_O3' in conf.aqi: IdeVec = (np.array([1, 1]), np.array([1, 2])) elif 'SURF_ug_NOx' in conf.aqi: IdeVec = (np.array([1, 1]), np.array([1, 2])) elif conf.domain == 'ineris7km': IdeVec = (np.array([1, 2]), np.array([1, 3]), np.array([1, 4]), np.array([1, 5]), np.array([1, 6])) #loop over precursors for precursor in range(0, nPrec): PREC = precursor Ide = IdeVec[precursor] icel = 0 #intialize variables numcells = nx * ny numcells = np.sum( flagRegioMat > 0) # create empty matrix only for really needed points PrecPatch = np.zeros((numcells, (rad * 2 + 1)**2)) IndicEq = np.zeros((numcells, 1)) latVec = np.zeros((numcells, 1)) indexUsed = np.full((numcells, 1), np.nan) #np.zeros((nx*ny,1)); potency = np.full((numcells), np.nan) #np.zeros((ny,nx)); print('precursor: ' + str(PREC)) for ic in range(0, nx): #print(PREC, ic); for ir in range(0, ny): if flagRegioMat[ir, ic] > 0: #variable to store which group ot be considered # indexUsed[icel] = np.where(val==potency[ir,ic]); #create data for omega calculation nSc = Ide.shape[0] - 1 # size(Ide,2)-1 tmpPrec = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide[1], PREC], rad) # patches tmpInde = ei.EquaIndic(ic, ir, rf, nx, ny, nSc, Indic[:, :, Ide[1]]) # indicator #store data for omega calculation PrecPatch[icel, :] = tmpPrec #np.squeeze(tmpPrec) IndicEq[icel] = tmpInde latVec[icel] = lat[ir, ic] icel = icel + 1 # indexUsedLin = np.reshape(indexUsed, -1, order='F'); #compute omega for each group of cells, given precursor p # for i in range(val.size): remInd = (IndicEq > 0).flatten() i = 1 x0 = [1, 1.5] # ind = np.where(indexUsedLin==i)[0]; inp1 = PrecPatch[remInd] #[ind,:]; inp2 = IndicEq[remInd] #[ind]; latVecFilt = latVec[remInd] bnds = ((0, 2), (0.1, 2.9)) opts = {'disp': False, 'ftol': 10e-9} mdl = minimize(iop, x0, args=(inp1, inp2, rad, latVecFilt, conf.ratioPoly), bounds=bnds, method='L-BFGS-B', options=opts) # L-BFGS-B, TNC # iop = lambda inp1,beta1,beta2: inv.InvDistN_opt_prec([beta1,beta2],inp1,rad); # # [mdl,r,J,CovB] = nlin.nlinfit(iop,inp1,inp2.ravel(),x0); # ci2[PREC] = nlpa.nlparci(r,J); # CovB2[PREC] = CovB; alphaTmp = mdl.x[0] omegaTmp = mdl.x[1] #repeat result for each belonging to a given group for ic in range(0, nx): for ir in range(0, ny): if flagRegioMat[ir, ic] > 0: # indexUsed = np.where(val==potency[ir,ic])[0]; alpha[ir, ic, PREC] = alphaTmp omega[ir, ic, PREC] = omegaTmp del (PrecPatch, IndicEq, indexUsed, potency) #rescale to initial spatial resolution, through nearest interpolation #initialize variable omegaFinal = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5)) alphaFinal = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5)) [i, j] = np.where(np.isfinite(omega[:, :, 0])) for p in range(0, nPrec): omegaFinal[:, :, p] = omega[i[0], j[0], p] alphaFinal[:, :, p] = alpha[i[0], j[0], p] #store final results conf.omegaFinalStep1_alldom = omegaFinal conf.alphaFinalStep1_alldom = alphaFinal conf.ci2Step1 = ci2 conf.CovB2Step1 = CovB2 ####################################################### ####################################################### #START NOW THE SECOND STEP OF THE COMPUTATION OF OMEGA ####################################################### ####################################################### # convert from 28 to 7 km Prec = f7.from7to28(conf.Prec) ny = int(conf.ny / 4) nx = int(conf.nx / 4) rad = conf.radStep2 nPrec = conf.nPrec rf = conf.rf1 flagRegioMat = np.copy(conf.flagRegioMat) # Y, X = np.mgrid[-rad:rad + 1:1, -rad:rad + 1:1]; # np.meshgrid(r_[-rad:1:rad], r_[-rad:1:rad]); # F = 1 / ((1 + (X ** 2 + Y ** 2) ** (0.5))); # pad Prec with zeros around initial matrix, to perform matrix products later on Prec2 = np.zeros( (ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3])) Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :] Prec = Prec2 # convert from 28 to 7 km Indic = f7.from7to28(conf.Indic) flagRegioMat = f7.from7to28(flagRegioMat) af = conf.alphaFinalStep1_alldom of = conf.omegaFinalStep1_alldom ICvecalpha = [ af[np.isfinite(af)][0], af[np.isfinite(af)][1], af[np.isfinite(af)][2], af[np.isfinite(af)][3], af[np.isfinite(af)][4] ] ICvecomega = [ of[np.isfinite(of)][0], of[np.isfinite(of)][1], of[np.isfinite(of)][2], of[np.isfinite(of)][3], of[np.isfinite(of)][4] ] # initialize variables omega = np.full([ny, nx, nPrec], 1.5) # omega = np.full([ny, nx, nPrec], 2); # for i in range(0,5): # omega[:,:,i] = of[np.isfinite(of)][i] ICvecalpha = [1, 1, 1, 1, 1] ICvecomega = [1.5, 1.5, 1.5, 1.5, 1.5] # loop over precursors for precursor in range(0, nPrec): PREC = precursor Ide = IdeVec[precursor] # intialize variables print('precursor: ' + str(PREC)) # icel=1 # flagIC = 0 t1 = time.time() #Ide or conf.Ide ??? argslist = (zip( np.where(flagRegioMat > 0)[0], np.where(flagRegioMat > 0)[1], repeat(Ide), repeat(rf), repeat(nx), repeat(ny), repeat(Prec), repeat(PREC), repeat(Indic), repeat(ICvecalpha), repeat(ICvecomega), repeat(rad), repeat(conf.ratioPoly), repeat(lat))) pool = mp.Pool() # by default use available corse print('***** Using parallel computing with ' + str(mp.cpu_count()) + ' cores *****') result = pool.starmap_async(computeOutput, argslist) pool.close() pool.join() print(str(time.time() - t1)) res = np.vstack(result.get()) # result as nparray print(res) alpha[np.where(flagRegioMat > 0)[0], np.where(flagRegioMat > 0)[1], PREC] = res[:, 0] omega[np.where(flagRegioMat > 0)[0], np.where(flagRegioMat > 0)[1], PREC] = res[:, 1] conf.omegaFinalStep1_notFiltered = omega omegaFinal2 = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5)) #from aggregated to initial resolution for i in range(0, nPrec): for irAgg in range(0, ny): for icAgg in range(0, nx): omegaFinal2[irAgg * 4:irAgg * 4 + 4, icAgg * 4:icAgg * 4 + 4, i] = omega[irAgg, icAgg, i] print('precursor interpolated: ' + str(i)) #initialize matrix and if needed apply gaussian filter omegaFinal = np.zeros_like(omegaFinal2) for i in range(0, 5): tmp = omegaFinal2[:, :, i] if conf.gf == 1: # if gaussian filter has to be applied tmp = gf.gaussian_filter(tmp, sigma=10) omegaFinal[:, :, i] = tmp #rount results and save it to final matrix omegaFinal = np.round(omegaFinal, 1) conf.omegaFinalStep1 = omegaFinal conf.ci2Step1 = [] conf.CovB2Step1 = []
def step1_omegaOptimization(conf): #convert from 28 to 7 km Prec = f7.from7to28(conf.Prec); ny = int(conf.ny/4); nx = int(conf.nx/4); rad = conf.radStep1; nPrec = 5#len(conf.vec3[conf.POLLSEL])#conf.nPrec; rf = conf.rf1 flagRegioMat = np.copy(conf.flagRegioMat); #pad Prec with zeros around initial matrix, to perform matrix products later on Prec2 = np.zeros((ny+rad*2,nx+rad*2,Prec.shape[2],Prec.shape[3])); Prec2[rad:-rad,rad:-rad,:,:] = Prec[:,:,:,:]; Prec=Prec2; #convert from 28 to 7 km Indic = f7.from7to28(conf.Indic); flagRegioMat = f7.from7to28(flagRegioMat); lat = f7.from7to28(conf.y); # flagPerNoxPP??m = f7.from7to28(flagPerNoxPPm); #initialize variables omega = np.full([ny,nx,nPrec],1.5); alpha = np.full([ny,nx,nPrec],np.nan); ci2 = np.empty((nPrec), dtype=object); CovB2 = np.empty((nPrec), dtype=object); # alphaTmp = np.zeros((categories.size)); # omegaTmp = np.zeros((categories.size)); #define training scenarios; note scenarios number is +1 if checking DoE...as in line 74 it is -1 if conf.domain == 'emep10km': if conf.aqi == 'SURF_ug_PM25_rh50-Yea': IdeVec = (np.array([1, 1]),np.array([1, 2]),np.array([1, 3]),np.array([1, 5]),np.array([1, 6])); elif conf.aqi == 'SURF_ug_PM10_rh50-Yea': IdeVec = (np.array([1, 1]),np.array([1, 2]),np.array([1, 3]),np.array([1, 4]),np.array([1, 6])); elif conf.domain == 'ineris7km': IdeVec = (np.array([1, 2]),np.array([1, 3]),np.array([1, 4]),np.array([1, 5]),np.array([1, 6])); elif (conf.domain == 'emepV433_camsV221') | (conf.domain == 'edgar2015') | (conf.domain == 'emepV434_camsV42'): IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]), np.array([1, 4]), np.array([1, 5])); #loop over precursors for precursor in range(0, 5): PREC = precursor; Ide = IdeVec[precursor]; # icel = 0; if (PREC == 2 or PREC== 3): bnds = ((0, 1), (1.5, 2.5)) else: bnds = ((0, 1), (1.5, 2.5)) #intialize variables # numcells = nx*ny # numcells = np.sum(flagRegioMat>0) # create empty matrix only for really needed points # PrecPatch = np.zeros((numcells,(rad*2+1)**2)); # IndicEq = np.zeros((numcells,1)); # latVec = np.zeros((numcells,1)); print('precursor: '+str(PREC)); for ic in range(0, nx): print(PREC, ic); for ir in range(0, ny): if flagRegioMat[ir,ic]>0: #variable to store which group ot be considered # indexUsed[icel] = np.where(val==potency[ir,ic]); #create data for omega calculation nSc = Ide.shape[0]-1;# size(Ide,2)-1 tmpPrec = ep.EquaPrec(ic,ir,rf,nx,ny,nSc,Prec.shape[3],Prec[:,:,Ide[1],PREC],rad); # patches tmpInde = ei.EquaIndic(ic,ir,rf,nx,ny,nSc,Indic[:,:,Ide[1]]); # indicator #store data for omega calculation # PrecPatch[icel,:] = tmpPrec; #np.squeeze(tmpPrec) # IndicEq[icel] = tmpInde; latVec = lat[ir,ic] # icel = icel+1; # remInd = (tmpInde>0).flatten() i=1 x0 = [1, 2]; # print(remInd) # inp1 = tmpPrec[remInd]#[ind,:]; # inp2 = tmpInde[remInd]#[ind]; inp1 = tmpPrec#[ind,:]; inp2 = tmpInde#[ind]; # mdl = minimize(iop, x0, args=(inp1, inp2, rad, latVec, conf.ratioPoly), bounds=bnds, method='BFGS', options=opts) # L-BFGS-B, TNC opts = {'disp': False} # mdl = minimize(iop, x0, args=(inp1, inp2, rad, latVec, conf.ratioPoly), method='BFGS', options=opts) # L-BFGS-B, TNC # print(mdl.x) mdl = minimize(iop, x0, args=(inp1, inp2, rad, latVec, conf.ratioPoly), bounds=bnds, method='L-BFGS-B', options=opts) # L-BFGS-B, TNC # print(mdl.x) # mdl = minimize(iop, x0, args=(inp1, inp2, rad, latVec, conf.ratioPoly), bounds=bnds, method='SLSQP', options=opts) # L-BFGS-B, TNC # print(mdl.x) alpha[ir,ic,PREC] = mdl.x[0]; omega[ir,ic,PREC] = mdl.x[1]; #rescale to initial spatial resolution, through nearest interpolation #initialize variable conf.omegaFinalStep1_notFiltered = omega omegaFinal2 = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5)); for i in range(0, nPrec): for irAgg in range(0, ny): for icAgg in range(0, nx): omegaFinal2[irAgg * 4:irAgg * 4 + 4, icAgg * 4:icAgg * 4 + 4, i] = omega[irAgg, icAgg, i] print('precursor interpolated: ' + str(i)); # omegaFinal = omegaFinal2 omegaFinal = np.zeros_like(omegaFinal2) for i in range(0, 5): tmp = omegaFinal2[:, :, i] tmp = gf.gaussian_filter(tmp, sigma=5) omegaFinal[:, :, i] = tmp omegaFinal = np.round(omegaFinal, 1) # omegaFinal = q.quant(omegaFinal, 0.2) # discretize #keep only results on the mask omegaFinal[conf.flagRegioMat==0] =np.nan conf.omegaFinalStep1 = omegaFinal; conf.ci2Step1 = []; conf.CovB2Step1 = [];
def step1_omegaOptimization(conf): prctileVec1 = np.array([100, 100, 100, 100, 100]) # prctileVec2=np.array([70, 70, 70, 70, 70]); categories = np.array([1]) #convert from 28 to 7 km Prec = f7.from7to28(conf.Prec) ny = int(conf.ny / 4) nx = int(conf.nx / 4) rad = conf.radStep1 nPrec = len(conf.vec3[conf.POLLSEL]) #conf.nPrec; rf = 0 flagRegioMat = np.copy(conf.flagRegioMat) #pad Prec with zeros around initial matrix, to perform matrix products later on Prec2 = np.zeros( (ny + rad * 2, nx + rad * 2, Prec.shape[2], Prec.shape[3])) Prec2[rad:-rad, rad:-rad, :, :] = Prec[:, :, :, :] Prec = Prec2 #convert from 28 to 7 km Indic = f7.from7to28(conf.Indic) flagRegioMat = f7.from7to28(flagRegioMat) lat = f7.from7to28(conf.y) # flagPerNoxPP??m = f7.from7to28(flagPerNoxPPm); #initialize variables omega = np.full([ny, nx, nPrec], np.nan) alpha = np.full([ny, nx, nPrec], np.nan) ci2 = np.empty((nPrec), dtype=object) CovB2 = np.empty((nPrec), dtype=object) # alphaTmp = np.zeros((categories.size)); # omegaTmp = np.zeros((categories.size)); #define training scenarios; note scenarios number is +1 if checking DoE...as in line 74 it is -1 if conf.domain == 'emep10km': if conf.aqi == 'SURF_ug_PM25_rh50-Yea': IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]), np.array([1, 5]), np.array([1, 6])) elif conf.aqi == 'SURF_ug_PM10_rh50-Yea': IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]), np.array([1, 4]), np.array([1, 6])) elif conf.domain == 'ineris7km': IdeVec = (np.array([1, 2]), np.array([1, 3]), np.array([1, 4]), np.array([1, 5]), np.array([1, 6])) elif (conf.domain == 'emepV433_camsV221') | ( conf.domain == 'edgar2015') | (conf.domain == 'emepV434_camsV42'): IdeVec = (np.array([1, 1]), np.array([1, 2]), np.array([1, 3]), np.array([1, 4]), np.array([1, 5])) #loop over precursors for precursor in range(0, nPrec): PREC = precursor Ide = IdeVec[precursor] icel = 0 #intialize variables numcells = nx * ny numcells = np.sum( flagRegioMat > 0) # create empty matrix only for really needed points PrecPatch = np.zeros((numcells, (rad * 2 + 1)**2)) IndicEq = np.zeros((numcells, 1)) latVec = np.zeros((numcells, 1)) indexUsed = np.full((numcells, 1), np.nan) #np.zeros((nx*ny,1)); potency = np.full((numcells), np.nan) #np.zeros((ny,nx)); print('precursor: ' + str(PREC)) for ic in range(0, nx): #print(PREC, ic); for ir in range(0, ny): if flagRegioMat[ir, ic] > 0: #variable to store which group ot be considered # indexUsed[icel] = np.where(val==potency[ir,ic]); #create data for omega calculation nSc = Ide.shape[0] - 1 # size(Ide,2)-1 tmpPrec = ep.EquaPrec(ic, ir, rf, nx, ny, nSc, Prec.shape[3], Prec[:, :, Ide[1], PREC], rad) # patches tmpInde = ei.EquaIndic(ic, ir, rf, nx, ny, nSc, Indic[:, :, Ide[1]]) # indicator #store data for omega calculation PrecPatch[icel, :] = tmpPrec #np.squeeze(tmpPrec) IndicEq[icel] = tmpInde latVec[icel] = lat[ir, ic] icel = icel + 1 # indexUsedLin = np.reshape(indexUsed, -1, order='F'); #compute omega for each group of cells, given precursor p # for i in range(val.size): remInd = (IndicEq > 0).flatten() i = 1 x0 = [1, 2] # ind = np.where(indexUsedLin==i)[0]; inp1 = PrecPatch[remInd] #[ind,:]; inp2 = IndicEq[remInd] #[ind]; latVecFilt = latVec[remInd] #rescaling input between min and max # scaler = MinMaxScaler(feature_range=(0, 1)) # scaler.fit(inp1) # inp1 = scaler.transform(inp1) # iop = lambda inp1,beta1,beta2: inv.InvDistN_opt_prec([beta1,beta2],inp1,rad); # [mdl,r,J,CovB] = nlin.nlinfit(iop,inp1,inp2.ravel(),x0); bnds = ((0, 2), (0.1, 2.9)) opts = {'disp': False, 'ftol': 10e-6} mdl = minimize(iop, x0, args=(inp1, inp2, rad, latVecFilt, conf.ratioPoly), bounds=bnds, method='L-BFGS-B', options=opts) # L-BFGS-B, TNC # ?import scipy.optimize.dif as bs #mdl = bs(iop, x0, args=(inp1, inp2, rad, latVecFilt, conf.ratioPoly), bounds=bnds, method='L-BFGS-B', options=opts) # L-BFGS-B, TNC # print('prec' + str(precursor)) # print(mdl) # ci2[PREC] = nlpa.nlparci(r,J); # CovB2[PREC] = CovB; alphaTmp = mdl.x[0] omegaTmp = mdl.x[1] #print(alphaTmp) #repeat result for each belonging to a given group for ic in range(0, nx): for ir in range(0, ny): if flagRegioMat[ir, ic] > 0: # indexUsed = np.where(val==potency[ir,ic])[0]; alpha[ir, ic, PREC] = alphaTmp omega[ir, ic, PREC] = omegaTmp del (PrecPatch, IndicEq, indexUsed, potency) #rescale to initial spatial resolution, through nearest interpolation #initialize variable omegaFinal = np.zeros((conf.Prec.shape[0], conf.Prec.shape[1], 5)) for i in range(0, 5): omegaFinal[:, :, i] = np.unique(omega[:, :, i])[0] #loop on precursors # for i in range(0, nPrec): # #define interpolator object # xgv = np.arange(1., conf.Prec.shape[0]/4+1); # ygv = np.arange(1., conf.Prec.shape[1]/4+1); # F=interpol.RegularGridInterpolator((xgv, ygv), omega[:,:,i],method='nearest',bounds_error=False, fill_value=None); # # #interpolate # Xq = np.arange(1., conf.Prec.shape[0]/4+1, 1/4); # Yq = np.arange(1., conf.Prec.shape[1]/4+1, 1/4); # [Y2,X2] = np.meshgrid(Yq, Xq); # pts=((X2.flatten(),Y2.flatten())) # omegaFinal[:,:,i] = F(pts).reshape(conf.Prec.shape[0],conf.Prec.shape[1]) # print('precursor interpolated: '+str(i)); #store final results # replacingVal = np.unique(omegaFinal[:,:,whichPollToUpdate][~np.isnan(omegaFinal[:,:,whichPollToUpdate])]) # conf.omegaFinalStep1[:,:,whichPollToUpdate] = replacingVal#omegaFinal[:,:,whichPollToUpdate]; conf.omegaFinalStep1 = omegaFinal # conf.omegaFinalStep1_28km = omegaFinal conf.ci2Step1 = ci2 conf.CovB2Step1 = CovB2