def eof(x, svd=False, transform=False): ''' Empirical Orthogonal Function (EOF) analysis to finds both time series and spatial patterns. :param x: (*array_like*) Input 2-D array with space-time field. :param svd: (*boolean*) Using SVD or eigen method. :param transform: (*boolean*) Do space-time transform or not. This transform will speed up the computation if the space location number is much more than time stamps. Only valid when ``svd=False``. :returns: (EOF, E, PC) EOF: eigen vector 2-D array; E: eigen values 1-D array; PC: Principle component 2-D array. ''' has_nan = False if x.contains_nan(): #Has NaN value valid_idx = np.where(x[:, 0] != np.nan)[0] xx = x[valid_idx, :] has_nan = True else: xx = x m, n = xx.shape if svd: U, S, V = np.linalg.svd(xx) EOF = U C = np.zeros((m, n)) for i in range(len(S)): C[i, i] = S[i] PC = np.dot(C, V) E = S**2 / n else: if transform: C = np.dot(xx.T, xx) E1, EOF1 = np.linalg.eig(C) EOF1 = EOF1[:, ::-1] E = E1[::-1] EOFa = np.dot(xx, EOF1) EOF = np.zeros((m, n)) for i in range(n): EOF[:, i] = EOFa[:, i] / np.sqrt(abs(E[i])) PC = np.dot(EOF.T, xx) else: C = np.dot(xx, xx.T) / n E, EOF = np.linalg.eig(C) PC = np.dot(EOF.T, xx) EOF = EOF[:, ::-1] PC = PC[::-1, :] E = E[::-1] if has_nan: _EOF = np.ones(x.shape) * np.nan _PC = np.ones(x.shape) * np.nan _EOF[valid_idx, :] = -EOF _PC[valid_idx, :] = -PC return _EOF, E, _PC else: return EOF, E, PC
def eof(x, transform=False): ''' Empirical Orthogonal Function (EOF) analysis to finds both time series and spatial patterns. :param x: (*array_like*) Input 2-D array with space-time field. :param transform: (*boolean*) Do space-time transform or not. This transform will speed up the computation if the space location number is much more than time stamps. :returns: (EOF, E, PC) EOF: eigen vector 2-D array; E: eigen values 1-D array; PC: Principle component 2-D array. ''' m, n = x.shape if transform: C = np.dot(x.T, x) E1, EOF1 = np.linalg.eig(C) EOF1 = EOF1[:, ::-1] E = E1[::-1] EOFa = np.dot(x, EOF1) EOF = np.zeros((m, n)) for i in range(n): EOF[:, i] = EOFa[:, i] / np.sqrt(abs(E[i])) PC = np.dot(EOF.T, x) PC = PC[::-1, :] else: C = np.dot(x, x.T) / n E, EOF = np.linalg.eig(C) PC = np.dot(EOF.T, x) EOF = EOF[:, ::-1] PC = PC[::-1, :] E = E[::-1] return EOF, E, PC
def grid_expand(data, emis_grid): """ Emission data grid expand to avoid NaN values in interpolation process. :param data: (*array*) emission data array (2D). :param emis_grid: (*GridSet*) emission grid set. :returns: Expanded data array and emission grid set. """ ny, nx = data.shape rdata = np.zeros((ny + 2, nx + 2)) rdata[1:-1, 1:-1] = data rdata[0, 1:-1] = data[0, :] rdata[-1, 1:-1] = data[-1, :] rdata[1:-1, 0] = data[:, 0] rdata[1:-1, -1] = data[:, -1] rdata[0, 0] = data[0, 0] rdata[-1, 0] = data[-1, 0] rdata[0, -1] = data[0, -1] rdata[-1, -1] = data[-1, -1] x = emis_grid.x_orig - emis_grid.x_delta y = emis_grid.y_orig - emis_grid.y_delta rgrid = GridDesc(emis_grid.proj, x_orig=x, x_cell=emis_grid.x_cell, x_num=emis_grid.x_num + 2, y_orig=y, y_cell=emis_grid.y_cell, y_num=emis_grid.y_num + 2) return rdata, rgrid
def week_allocation(data, week_profile, year, month, weekend_or_not=True): """ Weekly allocation. :param data: (*array*) Monthly emission data array - 2D. :param week_profile: (*WeekProfile*) Weekly profile. :param year: (*int*) The year. :param month: (*int*) The month. :param weekend_or_not: (*bool*) Only use weekend or not for allocation. :return: (*array*) Daily emission data array in a week - 3D. """ mdays = calendar.monthrange(year, month)[1] if weekend_or_not: weekend_days = get_weekend_days(year, month) weekday_days = mdays - weekend_days total_weight = week_profile.weekday_weight * weekday_days + week_profile.weekend_weight * \ weekend_days wdata = data / total_weight return wdata * week_profile.weekday_weight, wdata * week_profile.weekend_weight else: week_days = get_week_days(year, month) total_weight = (week_profile.weights * week_days).sum() wdata = data / total_weight ny, nx = data.shape d_data = np.zeros((7, ny, nx)) for i in range(7): d_data[i] = wdata * week_profile.weights[i] return d_data
def _learn(self): distance = EuclideanDistance() centers = np.zeros((50, self._x.shape[1]), dtype='double').tojarray('double') x = self._x.tojarray('double') rbf = SmileUtils.learnGaussianRadialBasis(x, centers) self._model = JRBFNetwork(x, self._y.tojarray('int'), distance, rbf, centers, self._normalized)
def speciation(data, pollutant_profile): """ Chemical speciation. :param data: (*array*) Pollutant data array. :param pollutant_profile: (*PollutantProfile*) The pollutant profile. :return: (*array*) Species data array. """ shape = list(data.shape) n = len(pollutant_profile) shape.insert(0, n) s_data = np.zeros(shape) for i in range(n): s_data[i] = data * pollutant_profile[i].mass_fraction return s_data
def month_allocation(data, month_profile): """ Monthly allocation. :param data: Yearly emission data array - 2D. :param month_profile: Monthly profile. :return: Monthly emission data array - 3D. """ ny, nx = data.shape m_data = np.zeros((12, ny, nx)) weights = month_profile.get_ratios() for i in range(12): m_data[i] = data * weights[i] return m_data
def diurnal_allocation(data, diurnal_profile): """ Diurnal allocation. :param data: Daily emission data array - 2D. :param diurnal_profile: Diurnal profile. :return: Hourly emission data array - 3D. """ ny, nx = data.shape h_data = np.zeros((24, ny, nx)) weights = diurnal_profile.get_ratios() for i in range(24): h_data[i] = data * weights[i] return h_data
def get_week_days(year, month): """ Get number of week days in a month. :param year: (*int*) The year. :param month: (*int*) The month. :return: (*array*) Number of week days. From Monday to Sunday. """ mdays = calendar.monthrange(year, month)[1] st = datetime.datetime(year, month, 1) et = datetime.datetime(year, month, mdays) wdays = np.zeros(7, 'int') while st <= et: i = st.weekday() wdays[i] = wdays[i] + 1 st = st + datetime.timedelta(days=1) return wdays
def makeshapes(x, y, type=None, z=None, m=None): """ Make shapes by x and y coordinates. :param x: (*array_like*) X coordinates. :param y: (*array_like*) Y coordinates. :param type: (*string*) Shape type [point | line | polygon]. :param z: (*array_like*) Z coordinates. :param m: (*array_like*) M coordinates. :returns: Shapes """ shapes = [] if isinstance(x, (int, float)): shape = PointShape() shape.setPoint(PointD(x, y)) shapes.append(shape) else: x = np.asarray(x)._array y = np.asarray(y)._array if not z is None: if m is None: m = np.zeros(len(z))._array else: m = np.asarray(m)._array z = np.asarray(z)._array if type == 'point': if z is None: shapes = ShapeUtil.createPointShapes(x, y) else: shapes = ShapeUtil.createPointShapes(x, y, z, m) elif type == 'line': if z is None: shapes = ShapeUtil.createPolylineShapes(x, y) else: shapes = ShapeUtil.createPolylineShapes(x, y, z, m) elif type == 'polygon': if z is None: shapes = ShapeUtil.createPolygonShapes(x, y) else: shapes = ShapeUtil.createPolygonShape(x, y, z, m) return shapes
def evaluate(self, points): """Evaluate the estimated pdf on a set of points. Parameters ---------- points : (# of dimensions, # of points)-array Alternatively, a (# of dimensions,) vector can be passed in and treated as a single point. Returns ------- values : (# of points,)-array The values at each point. Raises ------ ValueError : if the dimensionality of the input points is different than the dimensionality of the KDE. """ points = np.atleast_2d(points) dim, num_m = np.array(points).shape if dim != self.dim: raise ValueError("points have dimension {}, dataset has dimension " "{}".format(dim, self.dim)) result = np.zeros(num_m) if num_m >= self.num_dp: # there are more points than data, so loop over data for i in range(self.num_dp): diff = self.dataset[:, i, np.newaxis] - points tdiff = np.dot(self.inv_cov, diff) energy = np.sum(diff * tdiff, axis=0) / 2.0 result = result + np.exp(-energy) else: # loop over points for i in range(num_m): diff = self.dataset - points[:, i, np.newaxis] tdiff = np.dot(self.inv_cov, diff) energy = np.sum(diff * tdiff, axis=0) / 2.0 result[i] = np.sum(np.exp(-energy), axis=0) result = result / self.norm_factor return result
def predict(self, x): ''' Cluster a new instance to the nearest CF leaf. After building the CF tree, the user should call partition(int) method first to clustering leaves. Then they call this method to clustering new data. :param x: (*array*) A new instance. :returns: (*array*) the cluster label, which is the label of nearest CF leaf. Note that it may be Clustering.OUTLIER. ''' if x.ndim == 1: return self._model.predict(x) else: y = np.zeros(len(x), dtype='int') for i in range(len(x)): y[i] = self._model.predict(x[i]) return y ##############################################
def __init__(self, id="8", weights=None, weekday_weight=None, weekend_weight=None): """ Month profile :param id: (*str*) The id. :param weights: (*array*) The weights. Start from Monday. :param weekday_weight: (*float*) The weekday weight. :param weekend_weight: (*float*) The weekend weight. """ super(WeekProfile, self).__init__(id, weights) if self.weights is None: if weekday_weight is None or weekend_weight is None: weekday_weight = 147 weekend_weight = 132 self.weights = np.zeros(7) self.weights[:5] = weekday_weight self.weights[5:] = weekend_weight else: if isinstance(self.weights, (list, tuple)): self.weights = np.array(self.weights)
def cumsimp(y): """ Simpson-rule column-wise cumulative summation. Numerical approximation of a function F(x) such that Y(X) = dF/dX. Each column of the input matrix Y represents the value of the integrand Y(X) at equally spaced points X = 0,1,...size(Y,1). The output is a matrix F of the same size as Y. The first row of F is equal to zero and each following row is the approximation of the integral of each column of matrix Y up to the givem row. CUMSIMP assumes continuity of each column of the function Y(X) and uses Simpson rule summation. Similar to the command F = CUMSUM(Y), exept for zero first row and more accurate summation (under the assumption of continuous integrand Y(X)). Transferred from MATLAT code by Kirill K. Pankratov, March 7, 1994. :param y: (*array*) Input 2-D array. :returns: (*array*) Summation result. """ # 3-points interpolation coefficients to midpoints. # Second-order polynomial (parabolic) interpolation coefficients # from Xbasis = [0 1 2] to Xint = [.5 1.5] c1 = 3. / 8 c2 = 6. / 8 c3 = -1. / 8 # Determine the size of the input and make column if vector ist = 0 # If to be transposed lv = y.shape[0] if lv == 1: ist = 1 y = y.T lv = len(y) f = np.zeros(y.shape) # If only 2 elements in columns - simple sum divided by 2 if lv == 2: f[1, :] = (y[0, :] + y[1]) / 2 if ist: f = f.T # Transpose output if necessary return f # If more than two elements in columns - Simpson summation num = np.arange(0, lv - 2) # Interpolate values of Y to all midpoints f[num + 1, :] = c1 * y[num, :] + c2 * y[num + 1, :] + c3 * y[num + 2, :] f[num + 2, :] = f[ num + 2, :] + c3 * y[num, :] + c2 * y[num + 1, :] + c1 * y[num + 2, :] f[1, :] = f[1, :] * 2 f[lv - 1, :] = f[lv - 1, :] * 2 # Now Simpson (1,4,1) rule f[1:lv, :] = 2 * f[1:lv, :] + y[0:lv - 1, :] + y[1:lv, :] f = np.cumsum( f, axis=0) / 6 # Cumulative sum, 6 - denom. from the Simpson rule if ist: f = f.T # Transpose output if necessary return f
def run_transform(year, month, dir_inter, model_grid, out_species, out_species_aer, z): """ Distribution of particulate matter and change unit. :param year: (*int*) Year. :param month: (*int*) Month. :param dir_inter: (*string*) The directory where data is stored. :param model_grid: (*GridDesc*) Model data grid describe. :param out_species: (*list*) The name of the output species(gases and aerosol). :param out_species_aer: (*list*) The name of the output species(aerosol). :param z: (*int*) The zdim of the output data. """ print('Add input file...') fn_in = dir_inter + '\emis_{}_{}_hour.nc'.format(year, month) f_in = dataset.addfile(fn_in) #Set dimension print('Define dimensions and global attributes...') tdim = np.dimension(np.arange(24), 'Time') zdim = np.dimension(np.arange(z), 'emissions_zdim') ydim = np.dimension(model_grid.y_coord, 'south_north', 'Y') xdim = np.dimension(model_grid.x_coord, 'west_east', 'X') dims = [tdim, zdim, ydim, xdim] gattrs = OrderedDict() gattrs['Conventions'] = 'CF-1.6' gattrs['Tools'] = 'Created using MeteoInfo' #Set the definition of the output variable and ncfile fn_out = dir_inter + '\emis_{}_{}_hour_transform.nc'.format(year, month) print('Define variables and output file...') dimvars = [] for out_specie in out_species: dimvar = dataset.DimVariable() dimvar.name = out_specie dimvar.dtype = np.dtype.float dimvar.dims = dims dimvar.addattr('FieldType', '104') dimvar.addattr('MemoryOrder', "XYZ") dimvar.addattr('description', "EMISSION_{}".format(out_specie[2:])) if out_specie in out_species_aer: #g/m2/s to ug/m^3 m/s dimvar.addattr('units', 'ug/m3 m/s') else: #mole/m2/s to mol/km^2/hr dimvar.addattr('units', 'mol km^-2 hr^-1') dimvar.addattr('stagger', "") dimvar.addattr('coordinates', "XLONG XLAT XTIME") #dimvar.addattr('_ChunkSizes', '1U, 3U, 137U, 167U') dimvars.append(dimvar) ncfile = dataset.addfile(fn_out, 'c', largefile=True) ncfile.nc_define(dims, gattrs, dimvars) #add data to ncfile print('Process data and write to file...') for name in out_species: data = np.zeros((tdim.length, z, ydim.length, xdim.length)) sname = name[2:] print(sname) if sname in f_in.varnames(): data = f_in[sname][:, :, :] data = data * 3600 * 1e6 ncfile.write(name, data) elif sname == 'PM25I': data = f_in['PMFINE'][:, :, :] data = data * 1e6 ncfile.write(name, data * 0.2) elif sname == 'PM25J': data = f_in['PMFINE'][:, :, :] data = data * 1e6 ncfile.write(name, data * 0.8) elif sname == 'PM_10': data = f_in['PMC'][:, :, :] data = data * 1e6 ncfile.write(name, data) elif sname == 'ECI': data = f_in['PEC'][:, :, :] data = data * 1e6 ncfile.write(name, data * 0.2) elif sname == 'ECJ': data = f_in['PEC'][:, :, :] data = data * 1e6 ncfile.write(name, data * 0.8) elif sname == 'ORGI': data = f_in['POA'][:, :, :] data = data * 1e6 ncfile.write(name, data * 0.2) elif sname == 'ORGJ': data = f_in['POA'][:, :, :] data = data * 1e6 ncfile.write(name, data * 0.8) elif sname == 'SO4I': data = f_in['PSO4'][:, :, :] data = data * 1e6 ncfile.write(name, data * 0.2) elif sname == 'SO4J': data = f_in['PSO4'][:, :, :] data = data * 1e6 ncfile.write(name, data * 0.8) elif sname == 'NO3I': data = f_in['PNO3'][:, :, :] data = data * 1e6 ncfile.write(name, data * 0.2) elif sname == 'NO3J': data = f_in['PNO3'][:, :, :] data = data * 1e6 ncfile.write(name, data * 0.8) else: ncfile.write(name, data) f_in.close() ncfile.close() print('Distribution of particulate matter and change unit finised!')
def merge_output(year, months, model_grid, mechanism_name): """ Merge EMIPS output emission data of MEIC CAMS and HTAP data Applicable to the situation that MEIC data are nan outside China, and 0 in Taiwan!!! :param year: (*int*) Year. :param months: (*list*) Months. :param model_grid: (*GridDesc*) Model data grid describe. :param mechanism_name: (*string*) mechanism's name. """ print('-----------------------------------') print('----Merge output data(tw).....-----') print('-----------------------------------') #Set directories dir_meic1 = os.path.join(r'G:\test_data', mechanism_name, r'region_0.15\MEIC', str(year)) dir_cams1 = os.path.join(r'G:\test_data', mechanism_name, r'region_0.15\CAMS', str(year)) dir_htap1 = os.path.join(r'G:\test_data', mechanism_name, r'region_0.15\HTAP\2010') dir_out1 = os.path.join(r'G:\test_data', mechanism_name, r'region_0.15\merge', str(year)) #Set sectors sectors = [SectorEnum.INDUSTRY, SectorEnum.AGRICULTURE, SectorEnum.ENERGY, \ SectorEnum.RESIDENTIAL, SectorEnum.TRANSPORT, SectorEnum.SHIPS, \ SectorEnum.AIR] #Set dimensions tdim = np.dimension(np.arange(24), 'hour') ydim = np.dimension(model_grid.y_coord, 'lat', 'Y') xdim = np.dimension(model_grid.x_coord, 'lon', 'X') dims = [tdim, ydim, xdim] #Add data of taiwan area use CAMS/HTAP data tw = geolib.shaperead('taiwan.shp') for month in months: print('##########') print('Month: {}'.format(month)) print('##########') dir_meic = os.path.join(dir_meic1, '{}{:>02d}'.format(year, month)) dir_cams = os.path.join(dir_cams1, '{}{:>02d}'.format(year, month)) dir_htap = os.path.join(dir_htap1, '{}{:>02d}'.format(2010, month)) dir_out = os.path.join(dir_out1, '{}{:>02d}'.format(year, month)) if not os.path.exists(dir_out): os.mkdir(dir_out) print('------------------Filepath------------------') print('dir_meic: {}\ndir_cams: {}\ndir_htap: {}\ndir_out: {}'.format(dir_meic, dir_cams, dir_htap, dir_out)) #Sector loop for sector in sectors: print('############') print(sector) print('############') #MEIC data file fn_meic = os.path.join(dir_meic, 'emis_{}_{}_{}_hour.nc'.format(sector.name, year, month)) #CAMS data file fn_cams = os.path.join(dir_cams, 'emis_{}_{}_{}_hour.nc'.format(sector.name, year, month)) #HTAP data file fn_htap = os.path.join(dir_htap, 'emis_{}_{}_{}_hour.nc'.format(sector.name, 2010, month)) #Output data file fn_out = os.path.join(dir_out, 'emis_{}_{}_{}_hour.nc'.format(sector.name, year, month)) if os.path.exists(fn_meic) and (not os.path.exists(fn_cams)): shutil.copyfile(fn_meic, fn_out) print('Data from MEIC...') elif os.path.exists(fn_cams) and (not os.path.exists(fn_meic)): shutil.copyfile(fn_cams, fn_out) print('Data from CAMS...') else: f_meic = addfile(fn_meic) f_cams = addfile(fn_cams) f_htap = addfile(fn_htap) #Create output netcdf file ncfile = addfile(fn_out, 'c', largefile=True) #Set global attribute gattrs = dict(Conventions='CF-1.6', Tools='Created using MeteoInfo') #Get all variable dimvars = [] varnames = [] for var in f_meic.variables(): if var.ndim == 3: varnames.append(var.name) dimvar = DimVariable() dimvar.name = var.name dimvar.dtype = var.dtype dimvar.dims = dims dimvar.attributes = var.attributes dimvars.append(dimvar) for var in f_htap.variables(): if var.ndim == 3 and (not var.name in varnames): varnames.append(var.name) dimvar = DimVariable() dimvar.dtype = var.dtype dimvar.name = var.name dimvar.dims = dims dimvar.attributes = var.attributes dimvars.append(dimvar) #Define dimensions, global attributes and variables ncfile.nc_define(dims, gattrs, dimvars) #Write variable values for varname in varnames: print('\t{}'.format(varname)) data = None tda = np.zeros((24, len(model_grid.y_coord), len(model_grid.x_coord))) if varname in f_meic.varnames(): data = f_meic[varname][:] if varname in f_cams.varnames(): data1 = f_cams[varname][:] for i in range(24): tda[i] = data1[i].maskout(tw.shapes()) tda[tda==np.nan] = 0 if data is None: data = data1 else: mask = data.copy() mask[mask!=np.nan] = 0 mask[mask==np.nan] = 1 data[data==np.nan] = 0 data = data1 * mask + data data = data + tda elif varname in f_htap.varnames(): data1 = f_htap[varname][:] for i in range(24): tda[i] = data1[i].maskout(tw.shapes()) tda[tda==np.nan] = 0 if data is None: data = data1 else: mask = data.copy() mask[mask!=np.nan] = 0 mask[mask==np.nan] = 1 data[data==np.nan] = 0 data = data1 * mask + data data = data + tda ncfile.write(varname, data) #Close file f_meic.close() f_cams.close() f_htap.close() ncfile.close() print('---------------------------------------') print('-----Merge output data completed!------') print('---------------------------------------')
def run_proj(year, month, dir_inter, model_grid, target_grid, out_species, out_species_aer, global_attributes, z): """ Write Times variable, add global attributes, convert data's projection. io_style_emissions = 2 :param year: (*int*) Year. :param month: (*int*) Month. :param dir_inter: (*string*) The directory where data is stored. :param model_grid: (*GridDesc*) Model data grid describe. :param target_grid: (*GridDesc*) Target data grid describe. :param out_species: (*list*) The name of the output species(gases and aerosol). :param out_species_aer: (*list*) The name of the output species(aerosol). :param global_attributes: (*OrderedDict*) The global attributes of the output file. :param z: (*int*) The zdim of the output data. """ print('Add input file...') fn_in = dir_inter + '\emis_{}_{}_hour_transform.nc'.format(year, month) print(fn_in) f_in = dataset.addfile(fn_in) #set dimension tdim = np.dimension(np.arange(24), 'Time') ydim = np.dimension(target_grid.y_coord, 'south_north', 'Y') xdim = np.dimension(target_grid.x_coord, 'west_east', 'X') zdim = np.dimension(np.arange(z), 'emissions_zdim') sdim = np.dimension(np.arange(19), 'DateStrLen') dims = [tdim, zdim, ydim, xdim] all_dims = [tdim, sdim, xdim, ydim, zdim] fn_out = dir_inter + '\wrfchemi_d01_{}-{:0>2d}'.format(year, month) #set variables dimvars = [] dimvar = dataset.DimVariable() dimvar.name = 'Times' dimvar.dtype = np.dtype.char dimvar.dims = [tdim, sdim] #dimvar.addattr('_ChunkSizes', [1, 19]) dimvars.append(dimvar) for out_specie in out_species: dimvar = dataset.DimVariable() dimvar.name = out_specie dimvar.dtype = np.dtype.float dimvar.dims = dims dimvar.addattr('FieldType', 104) dimvar.addattr('MemoryOrder', "XYZ") dimvar.addattr('description', "EMISSION_{}".format(out_specie[2:])) if out_specie in out_species_aer: #g/m2/s to ug/m^3 m/s dimvar.addattr('units', 'ug/m3 m/s') else: #mole/m2/s to mol/km^2/hr dimvar.addattr('units', 'mol km^-2 hr^-1') dimvar.addattr('stagger', "") dimvar.addattr('coordinates', "XLONG XLAT XTIME") #dimvar.addattr('_ChunkSizes', [1, 3, 137, 167]) dimvars.append(dimvar) print('Create output data file...') print(fn_out) ncfile = dataset.addfile(fn_out, 'c', largefile=True) print('Define dimensions, global attributes and variables...') ncfile.nc_define(all_dims, global_attributes, dimvars, write_dimvars=False) #Times print('Write Times variable...') s_out = [] for i in range(24): s = '{}-{:0>2d}-01_{:0>2d}:00:00'.format(year, month, i) s_out.append(s) s_out = np.array(s_out, dtype=np.dtype.char) ncfile.write('Times', s_out) print('Write variable data except times...') for out_specie in out_species: data = np.zeros((tdim.length, zdim.length, ydim.length, xdim.length)) if out_specie in f_in.varnames(): print(out_specie) dd = f_in[out_specie][:] #Conversion proj dd = transform(dd, model_grid, target_grid) #Set default values dd[dd == np.nan] = 0 data[:, :, :, :] = dd ##########test############ #data[:, 1:8, :, :] = 0 ##########test############ ncfile.write(out_specie, data) f_in.close() ncfile.close() print('Convert projection finished!')
def run_merge(year, month, dir_inter, model_grid, sectors, z): """ Combine all sectors into one file :param year: (*int*) Year. :param month: (*int*) Month. :param dir_inter: (*string*) The directory where data is stored. :param model_grid: (*GridDesc*) Model data grid describe. :param sectors: (*list*) Sectors that needs to be merged. :param z: (*int*) The zdim of the output data. """ print('Define dimension and global attributes...') tdim = np.dimension(np.arange(24), 'hour') zdim = np.dimension(np.arange(z), 'emissions_zdim') ydim = np.dimension(model_grid.y_coord, 'lat', 'Y') xdim = np.dimension(model_grid.x_coord, 'lon', 'X') dims = [tdim, zdim, ydim, xdim] #Set the definition of the output variable print('Define variables...') dimvars = [] dict_spec = {} print('Add files...') for sector in sectors: fn = dir_inter + '\emis_{}_{}_{}_hour_height.nc'.format(sector, year, month) if os.path.exists(fn): print(fn) f = dataset.addfile(fn) for var in f.variables(): if var.ndim == 4: if dict_spec.has_key(var.name): dict_spec[var.name].append(fn) else: dict_spec[var.name] = [fn] for var in f.varnames(): if var == 'lat' or var == 'lon': continue else: dimvar = dataset.DimVariable() dimvar.name = var dimvar.dtype = np.dtype.float dimvar.dims = dims dimvar.addattr('description', "EMISSION_{}".format(var)) if var in out_species_aer: dimvar.addattr('units', 'g/m2/s') else: dimvar.addattr('units', 'mole/m2/s') dimvars.append(dimvar) f.close() else: print('File not exist: {}'.format(fn)) continue #Set dimension and define ncfile out_fn = dir_inter + '\emis_{}_{}_hour.nc'.format(year, month) gattrs = OrderedDict() gattrs['Conventions'] = 'CF-1.6' gattrs['Tools'] = 'Created using MeteoInfo' print('Create output data file...') ncfile = dataset.addfile(out_fn, 'c', largefile=True) ncfile.nc_define(dims, gattrs, dimvars) #read, merge and output print('Write variables data...') for sname, fns in dict_spec.iteritems(): print(sname) spec_data = np.zeros((tdim.length, z, ydim.length, xdim.length)) dd = np.zeros((tdim.length, z, ydim.length, xdim.length)) for fn in fns: f = dataset.addfile(fn) dd = f[sname][:] #turn nan to zero dd[dd==np.nan] = 0.0 if spec_data.sum() == 0.0: spec_data = dd else: spec_data = spec_data + dd f.close() ncfile.write(sname, spec_data) ncfile.close() print('Merge data finished!')
def run_allocate(year, month, dir_inter, model_grid, sectors, z): """ Assign data to different heights. If not specified, data is allocated to the first layer. :param year: (*int*) Year. :param month: (*int*) Month. :param dir_inter: (*string*) The directory where data is stored. :param model_grid: (*GridDesc*) Model data grid describe. :param sectors: (*GridDesc*) The sectors need to be processed. :param z: (*int*) The zdim of the output data. """ print('Define dimension and global attributes...') tdim = np.dimension(np.arange(24), 'hour') ydim = np.dimension(model_grid.y_coord, 'lat', 'Y') xdim = np.dimension(model_grid.x_coord, 'lon', 'X') zdim = np.dimension(np.arange(z), 'emissions_zdim') dims = [tdim, zdim, ydim, xdim] gattrs = OrderedDict() gattrs['Conventions'] = 'CF-1.6' gattrs['Tools'] = 'Created using MeteoInfo' for sector in sectors: fn = dir_inter + '\emis_{}_{}_{}_hour.nc'.format(sector, year, month) print('File input: {}'.format(fn)) dimvars = [] if os.path.exists(fn): f = dataset.addfile(fn) for var in f.varnames(): if var == 'lat' or var == 'lon': continue else: dimvar = dataset.DimVariable() dimvar.name = var dimvar.dtype = np.dtype.float dimvar.dims = dims dimvar.addattr('description', "EMISSION_{}".format(var)) if var in out_species_unit: dimvar.addattr('units', 'g/m2/s') else: dimvar.addattr('units', 'mole/m2/s') dimvars.append(dimvar) print('Create output data file...') out_fn = dir_inter + '\emis_{}_{}_{}_hour_height.nc'.format( sector, year, month) ncfile = dataset.addfile(out_fn, 'c', largefile=True) ncfile.nc_define(dims, gattrs, dimvars) data = np.zeros((tdim.length, z, ydim.length, xdim.length)) dd = np.zeros((tdim.length, z, ydim.length, xdim.length)) #read, merge and output if sector in sectors_al: print('Allocating: {}'.format(sector)) else: print('Do not need to be allocated: {}'.format(sector)) print('Write data to file...') for var in f.varnames(): if var == 'lat' or var == 'lon': continue else: print(var) dd[:, 0, :, :] = f[var][:] if sector in sectors_al: if sector == 'energy': data[:, 1, :, :] = dd[:, 0, :, :] * 0.1 data[:, 2, :, :] = dd[:, 0, :, :] * 0.1 data[:, 3, :, :] = dd[:, 0, :, :] * 0.3 data[:, 4, :, :] = dd[:, 0, :, :] * 0.2 data[:, 5, :, :] = dd[:, 0, :, :] * 0.2 data[:, 6, :, :] = dd[:, 0, :, :] * 0.1 if sector == 'industry': data[:, 0, :, :] = dd[:, 0, :, :] * 0.5 data[:, 1, :, :] = dd[:, 0, :, :] * 0.3 data[:, 2, :, :] = dd[:, 0, :, :] * 0.2 else: data[:, 0, :, :] = dd[:, 0, :, :] #Turn nan to zero data[data == np.nan] = 0 ###test### ''' if sector == 'energy': data[:, 0, :, :] = 0 ''' ###test### ncfile.write(var, data) ncfile.close() f.close() else: print('File not exist: {}'.format(fn)) continue print('Allocate of height finished!')