def eof(x, svd=False, transform=False):
'''
Empirical Orthogonal Function (EOF) analysis to finds both time series and spatial patterns.
:param x: (*array_like*) Input 2-D array with space-time field.
:param svd: (*boolean*) Using SVD or eigen method.
:param transform: (*boolean*) Do space-time transform or not. This transform will speed up
the computation if the space location number is much more than time stamps. Only valid
when ``svd=False``.
:returns: (EOF, E, PC) EOF: eigen vector 2-D array; E: eigen values 1-D array;
PC: Principle component 2-D array.
'''
has_nan = False
if x.contains_nan(): #Has NaN value
valid_idx = np.where(x[:, 0] != np.nan)[0]
xx = x[valid_idx, :]
has_nan = True
else:
xx = x
m, n = xx.shape
if svd:
U, S, V = np.linalg.svd(xx)
EOF = U
C = np.zeros((m, n))
for i in range(len(S)):
C[i, i] = S[i]
PC = np.dot(C, V)
E = S**2 / n
else:
if transform:
C = np.dot(xx.T, xx)
E1, EOF1 = np.linalg.eig(C)
EOF1 = EOF1[:, ::-1]
E = E1[::-1]
EOFa = np.dot(xx, EOF1)
EOF = np.zeros((m, n))
for i in range(n):
EOF[:, i] = EOFa[:, i] / np.sqrt(abs(E[i]))
PC = np.dot(EOF.T, xx)
else:
C = np.dot(xx, xx.T) / n
E, EOF = np.linalg.eig(C)
PC = np.dot(EOF.T, xx)
EOF = EOF[:, ::-1]
PC = PC[::-1, :]
E = E[::-1]
if has_nan:
_EOF = np.ones(x.shape) * np.nan
_PC = np.ones(x.shape) * np.nan
_EOF[valid_idx, :] = -EOF
_PC[valid_idx, :] = -PC
return _EOF, E, _PC
else:
return EOF, E, PC
def eof(x, transform=False):
'''
Empirical Orthogonal Function (EOF) analysis to finds both time series and spatial patterns.
:param x: (*array_like*) Input 2-D array with space-time field.
:param transform: (*boolean*) Do space-time transform or not. This transform will speed up
the computation if the space location number is much more than time stamps.
:returns: (EOF, E, PC) EOF: eigen vector 2-D array; E: eigen values 1-D array;
PC: Principle component 2-D array.
'''
m, n = x.shape
if transform:
C = np.dot(x.T, x)
E1, EOF1 = np.linalg.eig(C)
EOF1 = EOF1[:, ::-1]
E = E1[::-1]
EOFa = np.dot(x, EOF1)
EOF = np.zeros((m, n))
for i in range(n):
EOF[:, i] = EOFa[:, i] / np.sqrt(abs(E[i]))
PC = np.dot(EOF.T, x)
PC = PC[::-1, :]
else:
C = np.dot(x, x.T) / n
E, EOF = np.linalg.eig(C)
PC = np.dot(EOF.T, x)
EOF = EOF[:, ::-1]
PC = PC[::-1, :]
E = E[::-1]
return EOF, E, PC
def grid_expand(data, emis_grid):
"""
Emission data grid expand to avoid NaN values in interpolation process.
:param data: (*array*) emission data array (2D).
:param emis_grid: (*GridSet*) emission grid set.
:returns: Expanded data array and emission grid set.
"""
ny, nx = data.shape
rdata = np.zeros((ny + 2, nx + 2))
rdata[1:-1, 1:-1] = data
rdata[0, 1:-1] = data[0, :]
rdata[-1, 1:-1] = data[-1, :]
rdata[1:-1, 0] = data[:, 0]
rdata[1:-1, -1] = data[:, -1]
rdata[0, 0] = data[0, 0]
rdata[-1, 0] = data[-1, 0]
rdata[0, -1] = data[0, -1]
rdata[-1, -1] = data[-1, -1]
x = emis_grid.x_orig - emis_grid.x_delta
y = emis_grid.y_orig - emis_grid.y_delta
rgrid = GridDesc(emis_grid.proj,
x_orig=x,
x_cell=emis_grid.x_cell,
x_num=emis_grid.x_num + 2,
y_orig=y,
y_cell=emis_grid.y_cell,
y_num=emis_grid.y_num + 2)
return rdata, rgrid
def week_allocation(data, week_profile, year, month, weekend_or_not=True):
"""
Weekly allocation.
:param data: (*array*) Monthly emission data array - 2D.
:param week_profile: (*WeekProfile*) Weekly profile.
:param year: (*int*) The year.
:param month: (*int*) The month.
:param weekend_or_not: (*bool*) Only use weekend or not for allocation.
:return: (*array*) Daily emission data array in a week - 3D.
"""
mdays = calendar.monthrange(year, month)[1]
if weekend_or_not:
weekend_days = get_weekend_days(year, month)
weekday_days = mdays - weekend_days
total_weight = week_profile.weekday_weight * weekday_days + week_profile.weekend_weight * \
weekend_days
wdata = data / total_weight
return wdata * week_profile.weekday_weight, wdata * week_profile.weekend_weight
else:
week_days = get_week_days(year, month)
total_weight = (week_profile.weights * week_days).sum()
wdata = data / total_weight
ny, nx = data.shape
d_data = np.zeros((7, ny, nx))
for i in range(7):
d_data[i] = wdata * week_profile.weights[i]
return d_data
def _learn(self):
distance = EuclideanDistance()
centers = np.zeros((50, self._x.shape[1]), dtype='double').tojarray('double')
x = self._x.tojarray('double')
rbf = SmileUtils.learnGaussianRadialBasis(x, centers)
self._model = JRBFNetwork(x, self._y.tojarray('int'),
distance, rbf, centers, self._normalized)
def speciation(data, pollutant_profile):
"""
Chemical speciation.
:param data: (*array*) Pollutant data array.
:param pollutant_profile: (*PollutantProfile*) The pollutant profile.
:return: (*array*) Species data array.
"""
shape = list(data.shape)
n = len(pollutant_profile)
shape.insert(0, n)
s_data = np.zeros(shape)
for i in range(n):
s_data[i] = data * pollutant_profile[i].mass_fraction
return s_data
def month_allocation(data, month_profile):
"""
Monthly allocation.
:param data: Yearly emission data array - 2D.
:param month_profile: Monthly profile.
:return: Monthly emission data array - 3D.
"""
ny, nx = data.shape
m_data = np.zeros((12, ny, nx))
weights = month_profile.get_ratios()
for i in range(12):
m_data[i] = data * weights[i]
return m_data
def diurnal_allocation(data, diurnal_profile):
"""
Diurnal allocation.
:param data: Daily emission data array - 2D.
:param diurnal_profile: Diurnal profile.
:return: Hourly emission data array - 3D.
"""
ny, nx = data.shape
h_data = np.zeros((24, ny, nx))
weights = diurnal_profile.get_ratios()
for i in range(24):
h_data[i] = data * weights[i]
return h_data
def get_week_days(year, month):
"""
Get number of week days in a month.
:param year: (*int*) The year.
:param month: (*int*) The month.
:return: (*array*) Number of week days. From Monday to Sunday.
"""
mdays = calendar.monthrange(year, month)[1]
st = datetime.datetime(year, month, 1)
et = datetime.datetime(year, month, mdays)
wdays = np.zeros(7, 'int')
while st <= et:
i = st.weekday()
wdays[i] = wdays[i] + 1
st = st + datetime.timedelta(days=1)
return wdays
def makeshapes(x, y, type=None, z=None, m=None):
"""
Make shapes by x and y coordinates.
:param x: (*array_like*) X coordinates.
:param y: (*array_like*) Y coordinates.
:param type: (*string*) Shape type [point | line | polygon].
:param z: (*array_like*) Z coordinates.
:param m: (*array_like*) M coordinates.
:returns: Shapes
"""
shapes = []
if isinstance(x, (int, float)):
shape = PointShape()
shape.setPoint(PointD(x, y))
shapes.append(shape)
else:
x = np.asarray(x)._array
y = np.asarray(y)._array
if not z is None:
if m is None:
m = np.zeros(len(z))._array
else:
m = np.asarray(m)._array
z = np.asarray(z)._array
if type == 'point':
if z is None:
shapes = ShapeUtil.createPointShapes(x, y)
else:
shapes = ShapeUtil.createPointShapes(x, y, z, m)
elif type == 'line':
if z is None:
shapes = ShapeUtil.createPolylineShapes(x, y)
else:
shapes = ShapeUtil.createPolylineShapes(x, y, z, m)
elif type == 'polygon':
if z is None:
shapes = ShapeUtil.createPolygonShapes(x, y)
else:
shapes = ShapeUtil.createPolygonShape(x, y, z, m)
return shapes
def evaluate(self, points):
"""Evaluate the estimated pdf on a set of points.
Parameters
----------
points : (# of dimensions, # of points)-array
Alternatively, a (# of dimensions,) vector can be passed in and
treated as a single point.
Returns
-------
values : (# of points,)-array
The values at each point.
Raises
------
ValueError : if the dimensionality of the input points is different
than the dimensionality of the KDE.
"""
points = np.atleast_2d(points)
dim, num_m = np.array(points).shape
if dim != self.dim:
raise ValueError("points have dimension {}, dataset has dimension "
"{}".format(dim, self.dim))
result = np.zeros(num_m)
if num_m >= self.num_dp:
# there are more points than data, so loop over data
for i in range(self.num_dp):
diff = self.dataset[:, i, np.newaxis] - points
tdiff = np.dot(self.inv_cov, diff)
energy = np.sum(diff * tdiff, axis=0) / 2.0
result = result + np.exp(-energy)
else:
# loop over points
for i in range(num_m):
diff = self.dataset - points[:, i, np.newaxis]
tdiff = np.dot(self.inv_cov, diff)
energy = np.sum(diff * tdiff, axis=0) / 2.0
result[i] = np.sum(np.exp(-energy), axis=0)
result = result / self.norm_factor
return result
def predict(self, x):
'''
Cluster a new instance to the nearest CF leaf. After building the CF tree, the user should
call partition(int) method first to clustering leaves. Then they call this method to
clustering new data.
:param x: (*array*) A new instance.
:returns: (*array*) the cluster label, which is the label of nearest CF leaf. Note that it
may be Clustering.OUTLIER.
'''
if x.ndim == 1:
return self._model.predict(x)
else:
y = np.zeros(len(x), dtype='int')
for i in range(len(x)):
y[i] = self._model.predict(x[i])
return y
##############################################
def __init__(self,
id="8",
weights=None,
weekday_weight=None,
weekend_weight=None):
"""
Month profile
:param id: (*str*) The id.
:param weights: (*array*) The weights. Start from Monday.
:param weekday_weight: (*float*) The weekday weight.
:param weekend_weight: (*float*) The weekend weight.
"""
super(WeekProfile, self).__init__(id, weights)
if self.weights is None:
if weekday_weight is None or weekend_weight is None:
weekday_weight = 147
weekend_weight = 132
self.weights = np.zeros(7)
self.weights[:5] = weekday_weight
self.weights[5:] = weekend_weight
else:
if isinstance(self.weights, (list, tuple)):
self.weights = np.array(self.weights)
def cumsimp(y):
"""
Simpson-rule column-wise cumulative summation.
Numerical approximation of a function F(x) such that
Y(X) = dF/dX. Each column of the input matrix Y represents
the value of the integrand Y(X) at equally spaced points
X = 0,1,...size(Y,1).
The output is a matrix F of the same size as Y.
The first row of F is equal to zero and each following row
is the approximation of the integral of each column of matrix
Y up to the givem row.
CUMSIMP assumes continuity of each column of the function Y(X)
and uses Simpson rule summation.
Similar to the command F = CUMSUM(Y), exept for zero first
row and more accurate summation (under the assumption of
continuous integrand Y(X)).
Transferred from MATLAT code by Kirill K. Pankratov, March 7, 1994.
:param y: (*array*) Input 2-D array.
:returns: (*array*) Summation result.
"""
# 3-points interpolation coefficients to midpoints.
# Second-order polynomial (parabolic) interpolation coefficients
# from Xbasis = [0 1 2] to Xint = [.5 1.5]
c1 = 3. / 8
c2 = 6. / 8
c3 = -1. / 8
# Determine the size of the input and make column if vector
ist = 0 # If to be transposed
lv = y.shape[0]
if lv == 1:
ist = 1
y = y.T
lv = len(y)
f = np.zeros(y.shape)
# If only 2 elements in columns - simple sum divided by 2
if lv == 2:
f[1, :] = (y[0, :] + y[1]) / 2
if ist:
f = f.T # Transpose output if necessary
return f
# If more than two elements in columns - Simpson summation
num = np.arange(0, lv - 2)
# Interpolate values of Y to all midpoints
f[num + 1, :] = c1 * y[num, :] + c2 * y[num + 1, :] + c3 * y[num + 2, :]
f[num + 2, :] = f[
num + 2, :] + c3 * y[num, :] + c2 * y[num + 1, :] + c1 * y[num + 2, :]
f[1, :] = f[1, :] * 2
f[lv - 1, :] = f[lv - 1, :] * 2
# Now Simpson (1,4,1) rule
f[1:lv, :] = 2 * f[1:lv, :] + y[0:lv - 1, :] + y[1:lv, :]
f = np.cumsum(
f, axis=0) / 6 # Cumulative sum, 6 - denom. from the Simpson rule
if ist:
f = f.T # Transpose output if necessary
return f
def run_transform(year, month, dir_inter, model_grid, out_species,
out_species_aer, z):
"""
Distribution of particulate matter and change unit.
:param year: (*int*) Year.
:param month: (*int*) Month.
:param dir_inter: (*string*) The directory where data is stored.
:param model_grid: (*GridDesc*) Model data grid describe.
:param out_species: (*list*) The name of the output species(gases and aerosol).
:param out_species_aer: (*list*) The name of the output species(aerosol).
:param z: (*int*) The zdim of the output data.
"""
print('Add input file...')
fn_in = dir_inter + '\emis_{}_{}_hour.nc'.format(year, month)
f_in = dataset.addfile(fn_in)
#Set dimension
print('Define dimensions and global attributes...')
tdim = np.dimension(np.arange(24), 'Time')
zdim = np.dimension(np.arange(z), 'emissions_zdim')
ydim = np.dimension(model_grid.y_coord, 'south_north', 'Y')
xdim = np.dimension(model_grid.x_coord, 'west_east', 'X')
dims = [tdim, zdim, ydim, xdim]
gattrs = OrderedDict()
gattrs['Conventions'] = 'CF-1.6'
gattrs['Tools'] = 'Created using MeteoInfo'
#Set the definition of the output variable and ncfile
fn_out = dir_inter + '\emis_{}_{}_hour_transform.nc'.format(year, month)
print('Define variables and output file...')
dimvars = []
for out_specie in out_species:
dimvar = dataset.DimVariable()
dimvar.name = out_specie
dimvar.dtype = np.dtype.float
dimvar.dims = dims
dimvar.addattr('FieldType', '104')
dimvar.addattr('MemoryOrder', "XYZ")
dimvar.addattr('description', "EMISSION_{}".format(out_specie[2:]))
if out_specie in out_species_aer:
#g/m2/s to ug/m^3 m/s
dimvar.addattr('units', 'ug/m3 m/s')
else:
#mole/m2/s to mol/km^2/hr
dimvar.addattr('units', 'mol km^-2 hr^-1')
dimvar.addattr('stagger', "")
dimvar.addattr('coordinates', "XLONG XLAT XTIME")
#dimvar.addattr('_ChunkSizes', '1U, 3U, 137U, 167U')
dimvars.append(dimvar)
ncfile = dataset.addfile(fn_out, 'c', largefile=True)
ncfile.nc_define(dims, gattrs, dimvars)
#add data to ncfile
print('Process data and write to file...')
for name in out_species:
data = np.zeros((tdim.length, z, ydim.length, xdim.length))
sname = name[2:]
print(sname)
if sname in f_in.varnames():
data = f_in[sname][:, :, :]
data = data * 3600 * 1e6
ncfile.write(name, data)
elif sname == 'PM25I':
data = f_in['PMFINE'][:, :, :]
data = data * 1e6
ncfile.write(name, data * 0.2)
elif sname == 'PM25J':
data = f_in['PMFINE'][:, :, :]
data = data * 1e6
ncfile.write(name, data * 0.8)
elif sname == 'PM_10':
data = f_in['PMC'][:, :, :]
data = data * 1e6
ncfile.write(name, data)
elif sname == 'ECI':
data = f_in['PEC'][:, :, :]
data = data * 1e6
ncfile.write(name, data * 0.2)
elif sname == 'ECJ':
data = f_in['PEC'][:, :, :]
data = data * 1e6
ncfile.write(name, data * 0.8)
elif sname == 'ORGI':
data = f_in['POA'][:, :, :]
data = data * 1e6
ncfile.write(name, data * 0.2)
elif sname == 'ORGJ':
data = f_in['POA'][:, :, :]
data = data * 1e6
ncfile.write(name, data * 0.8)
elif sname == 'SO4I':
data = f_in['PSO4'][:, :, :]
data = data * 1e6
ncfile.write(name, data * 0.2)
elif sname == 'SO4J':
data = f_in['PSO4'][:, :, :]
data = data * 1e6
ncfile.write(name, data * 0.8)
elif sname == 'NO3I':
data = f_in['PNO3'][:, :, :]
data = data * 1e6
ncfile.write(name, data * 0.2)
elif sname == 'NO3J':
data = f_in['PNO3'][:, :, :]
data = data * 1e6
ncfile.write(name, data * 0.8)
else:
ncfile.write(name, data)
f_in.close()
ncfile.close()
print('Distribution of particulate matter and change unit finised!')
def merge_output(year, months, model_grid, mechanism_name):
"""
Merge EMIPS output emission data of MEIC CAMS and HTAP data
Applicable to the situation that MEIC data are nan outside China, and 0 in Taiwan!!!
:param year: (*int*) Year.
:param months: (*list*) Months.
:param model_grid: (*GridDesc*) Model data grid describe.
:param mechanism_name: (*string*) mechanism's name.
"""
print('-----------------------------------')
print('----Merge output data(tw).....-----')
print('-----------------------------------')
#Set directories
dir_meic1 = os.path.join(r'G:\test_data', mechanism_name, r'region_0.15\MEIC', str(year))
dir_cams1 = os.path.join(r'G:\test_data', mechanism_name, r'region_0.15\CAMS', str(year))
dir_htap1 = os.path.join(r'G:\test_data', mechanism_name, r'region_0.15\HTAP\2010')
dir_out1 = os.path.join(r'G:\test_data', mechanism_name, r'region_0.15\merge', str(year))
#Set sectors
sectors = [SectorEnum.INDUSTRY, SectorEnum.AGRICULTURE, SectorEnum.ENERGY, \
SectorEnum.RESIDENTIAL, SectorEnum.TRANSPORT, SectorEnum.SHIPS, \
SectorEnum.AIR]
#Set dimensions
tdim = np.dimension(np.arange(24), 'hour')
ydim = np.dimension(model_grid.y_coord, 'lat', 'Y')
xdim = np.dimension(model_grid.x_coord, 'lon', 'X')
dims = [tdim, ydim, xdim]
#Add data of taiwan area use CAMS/HTAP data
tw = geolib.shaperead('taiwan.shp')
for month in months:
print('##########')
print('Month: {}'.format(month))
print('##########')
dir_meic = os.path.join(dir_meic1, '{}{:>02d}'.format(year, month))
dir_cams = os.path.join(dir_cams1, '{}{:>02d}'.format(year, month))
dir_htap = os.path.join(dir_htap1, '{}{:>02d}'.format(2010, month))
dir_out = os.path.join(dir_out1, '{}{:>02d}'.format(year, month))
if not os.path.exists(dir_out):
os.mkdir(dir_out)
print('------------------Filepath------------------')
print('dir_meic: {}\ndir_cams: {}\ndir_htap: {}\ndir_out: {}'.format(dir_meic, dir_cams, dir_htap, dir_out))
#Sector loop
for sector in sectors:
print('############')
print(sector)
print('############')
#MEIC data file
fn_meic = os.path.join(dir_meic, 'emis_{}_{}_{}_hour.nc'.format(sector.name, year, month))
#CAMS data file
fn_cams = os.path.join(dir_cams, 'emis_{}_{}_{}_hour.nc'.format(sector.name, year, month))
#HTAP data file
fn_htap = os.path.join(dir_htap, 'emis_{}_{}_{}_hour.nc'.format(sector.name, 2010, month))
#Output data file
fn_out = os.path.join(dir_out, 'emis_{}_{}_{}_hour.nc'.format(sector.name, year, month))
if os.path.exists(fn_meic) and (not os.path.exists(fn_cams)):
shutil.copyfile(fn_meic, fn_out)
print('Data from MEIC...')
elif os.path.exists(fn_cams) and (not os.path.exists(fn_meic)):
shutil.copyfile(fn_cams, fn_out)
print('Data from CAMS...')
else:
f_meic = addfile(fn_meic)
f_cams = addfile(fn_cams)
f_htap = addfile(fn_htap)
#Create output netcdf file
ncfile = addfile(fn_out, 'c', largefile=True)
#Set global attribute
gattrs = dict(Conventions='CF-1.6', Tools='Created using MeteoInfo')
#Get all variable
dimvars = []
varnames = []
for var in f_meic.variables():
if var.ndim == 3:
varnames.append(var.name)
dimvar = DimVariable()
dimvar.name = var.name
dimvar.dtype = var.dtype
dimvar.dims = dims
dimvar.attributes = var.attributes
dimvars.append(dimvar)
for var in f_htap.variables():
if var.ndim == 3 and (not var.name in varnames):
varnames.append(var.name)
dimvar = DimVariable()
dimvar.dtype = var.dtype
dimvar.name = var.name
dimvar.dims = dims
dimvar.attributes = var.attributes
dimvars.append(dimvar)
#Define dimensions, global attributes and variables
ncfile.nc_define(dims, gattrs, dimvars)
#Write variable values
for varname in varnames:
print('\t{}'.format(varname))
data = None
tda = np.zeros((24, len(model_grid.y_coord), len(model_grid.x_coord)))
if varname in f_meic.varnames():
data = f_meic[varname][:]
if varname in f_cams.varnames():
data1 = f_cams[varname][:]
for i in range(24):
tda[i] = data1[i].maskout(tw.shapes())
tda[tda==np.nan] = 0
if data is None:
data = data1
else:
mask = data.copy()
mask[mask!=np.nan] = 0
mask[mask==np.nan] = 1
data[data==np.nan] = 0
data = data1 * mask + data
data = data + tda
elif varname in f_htap.varnames():
data1 = f_htap[varname][:]
for i in range(24):
tda[i] = data1[i].maskout(tw.shapes())
tda[tda==np.nan] = 0
if data is None:
data = data1
else:
mask = data.copy()
mask[mask!=np.nan] = 0
mask[mask==np.nan] = 1
data[data==np.nan] = 0
data = data1 * mask + data
data = data + tda
ncfile.write(varname, data)
#Close file
f_meic.close()
f_cams.close()
f_htap.close()
ncfile.close()
print('---------------------------------------')
print('-----Merge output data completed!------')
print('---------------------------------------')
def run_proj(year, month, dir_inter, model_grid, target_grid, out_species,
out_species_aer, global_attributes, z):
"""
Write Times variable, add global attributes, convert data's projection.
io_style_emissions = 2
:param year: (*int*) Year.
:param month: (*int*) Month.
:param dir_inter: (*string*) The directory where data is stored.
:param model_grid: (*GridDesc*) Model data grid describe.
:param target_grid: (*GridDesc*) Target data grid describe.
:param out_species: (*list*) The name of the output species(gases and aerosol).
:param out_species_aer: (*list*) The name of the output species(aerosol).
:param global_attributes: (*OrderedDict*) The global attributes of the output file.
:param z: (*int*) The zdim of the output data.
"""
print('Add input file...')
fn_in = dir_inter + '\emis_{}_{}_hour_transform.nc'.format(year, month)
print(fn_in)
f_in = dataset.addfile(fn_in)
#set dimension
tdim = np.dimension(np.arange(24), 'Time')
ydim = np.dimension(target_grid.y_coord, 'south_north', 'Y')
xdim = np.dimension(target_grid.x_coord, 'west_east', 'X')
zdim = np.dimension(np.arange(z), 'emissions_zdim')
sdim = np.dimension(np.arange(19), 'DateStrLen')
dims = [tdim, zdim, ydim, xdim]
all_dims = [tdim, sdim, xdim, ydim, zdim]
fn_out = dir_inter + '\wrfchemi_d01_{}-{:0>2d}'.format(year, month)
#set variables
dimvars = []
dimvar = dataset.DimVariable()
dimvar.name = 'Times'
dimvar.dtype = np.dtype.char
dimvar.dims = [tdim, sdim]
#dimvar.addattr('_ChunkSizes', [1, 19])
dimvars.append(dimvar)
for out_specie in out_species:
dimvar = dataset.DimVariable()
dimvar.name = out_specie
dimvar.dtype = np.dtype.float
dimvar.dims = dims
dimvar.addattr('FieldType', 104)
dimvar.addattr('MemoryOrder', "XYZ")
dimvar.addattr('description', "EMISSION_{}".format(out_specie[2:]))
if out_specie in out_species_aer:
#g/m2/s to ug/m^3 m/s
dimvar.addattr('units', 'ug/m3 m/s')
else:
#mole/m2/s to mol/km^2/hr
dimvar.addattr('units', 'mol km^-2 hr^-1')
dimvar.addattr('stagger', "")
dimvar.addattr('coordinates', "XLONG XLAT XTIME")
#dimvar.addattr('_ChunkSizes', [1, 3, 137, 167])
dimvars.append(dimvar)
print('Create output data file...')
print(fn_out)
ncfile = dataset.addfile(fn_out, 'c', largefile=True)
print('Define dimensions, global attributes and variables...')
ncfile.nc_define(all_dims, global_attributes, dimvars, write_dimvars=False)
#Times
print('Write Times variable...')
s_out = []
for i in range(24):
s = '{}-{:0>2d}-01_{:0>2d}:00:00'.format(year, month, i)
s_out.append(s)
s_out = np.array(s_out, dtype=np.dtype.char)
ncfile.write('Times', s_out)
print('Write variable data except times...')
for out_specie in out_species:
data = np.zeros((tdim.length, zdim.length, ydim.length, xdim.length))
if out_specie in f_in.varnames():
print(out_specie)
dd = f_in[out_specie][:]
#Conversion proj
dd = transform(dd, model_grid, target_grid)
#Set default values
dd[dd == np.nan] = 0
data[:, :, :, :] = dd
##########test############
#data[:, 1:8, :, :] = 0
##########test############
ncfile.write(out_specie, data)
f_in.close()
ncfile.close()
print('Convert projection finished!')
def run_merge(year, month, dir_inter, model_grid, sectors, z):
"""
Combine all sectors into one file
:param year: (*int*) Year.
:param month: (*int*) Month.
:param dir_inter: (*string*) The directory where data is stored.
:param model_grid: (*GridDesc*) Model data grid describe.
:param sectors: (*list*) Sectors that needs to be merged.
:param z: (*int*) The zdim of the output data.
"""
print('Define dimension and global attributes...')
tdim = np.dimension(np.arange(24), 'hour')
zdim = np.dimension(np.arange(z), 'emissions_zdim')
ydim = np.dimension(model_grid.y_coord, 'lat', 'Y')
xdim = np.dimension(model_grid.x_coord, 'lon', 'X')
dims = [tdim, zdim, ydim, xdim]
#Set the definition of the output variable
print('Define variables...')
dimvars = []
dict_spec = {}
print('Add files...')
for sector in sectors:
fn = dir_inter + '\emis_{}_{}_{}_hour_height.nc'.format(sector, year, month)
if os.path.exists(fn):
print(fn)
f = dataset.addfile(fn)
for var in f.variables():
if var.ndim == 4:
if dict_spec.has_key(var.name):
dict_spec[var.name].append(fn)
else:
dict_spec[var.name] = [fn]
for var in f.varnames():
if var == 'lat' or var == 'lon':
continue
else:
dimvar = dataset.DimVariable()
dimvar.name = var
dimvar.dtype = np.dtype.float
dimvar.dims = dims
dimvar.addattr('description', "EMISSION_{}".format(var))
if var in out_species_aer:
dimvar.addattr('units', 'g/m2/s')
else:
dimvar.addattr('units', 'mole/m2/s')
dimvars.append(dimvar)
f.close()
else:
print('File not exist: {}'.format(fn))
continue
#Set dimension and define ncfile
out_fn = dir_inter + '\emis_{}_{}_hour.nc'.format(year, month)
gattrs = OrderedDict()
gattrs['Conventions'] = 'CF-1.6'
gattrs['Tools'] = 'Created using MeteoInfo'
print('Create output data file...')
ncfile = dataset.addfile(out_fn, 'c', largefile=True)
ncfile.nc_define(dims, gattrs, dimvars)
#read, merge and output
print('Write variables data...')
for sname, fns in dict_spec.iteritems():
print(sname)
spec_data = np.zeros((tdim.length, z, ydim.length, xdim.length))
dd = np.zeros((tdim.length, z, ydim.length, xdim.length))
for fn in fns:
f = dataset.addfile(fn)
dd = f[sname][:]
#turn nan to zero
dd[dd==np.nan] = 0.0
if spec_data.sum() == 0.0:
spec_data = dd
else:
spec_data = spec_data + dd
f.close()
ncfile.write(sname, spec_data)
ncfile.close()
print('Merge data finished!')
def run_allocate(year, month, dir_inter, model_grid, sectors, z):
"""
Assign data to different heights.
If not specified, data is allocated to the first layer.
:param year: (*int*) Year.
:param month: (*int*) Month.
:param dir_inter: (*string*) The directory where data is stored.
:param model_grid: (*GridDesc*) Model data grid describe.
:param sectors: (*GridDesc*) The sectors need to be processed.
:param z: (*int*) The zdim of the output data.
"""
print('Define dimension and global attributes...')
tdim = np.dimension(np.arange(24), 'hour')
ydim = np.dimension(model_grid.y_coord, 'lat', 'Y')
xdim = np.dimension(model_grid.x_coord, 'lon', 'X')
zdim = np.dimension(np.arange(z), 'emissions_zdim')
dims = [tdim, zdim, ydim, xdim]
gattrs = OrderedDict()
gattrs['Conventions'] = 'CF-1.6'
gattrs['Tools'] = 'Created using MeteoInfo'
for sector in sectors:
fn = dir_inter + '\emis_{}_{}_{}_hour.nc'.format(sector, year, month)
print('File input: {}'.format(fn))
dimvars = []
if os.path.exists(fn):
f = dataset.addfile(fn)
for var in f.varnames():
if var == 'lat' or var == 'lon':
continue
else:
dimvar = dataset.DimVariable()
dimvar.name = var
dimvar.dtype = np.dtype.float
dimvar.dims = dims
dimvar.addattr('description', "EMISSION_{}".format(var))
if var in out_species_unit:
dimvar.addattr('units', 'g/m2/s')
else:
dimvar.addattr('units', 'mole/m2/s')
dimvars.append(dimvar)
print('Create output data file...')
out_fn = dir_inter + '\emis_{}_{}_{}_hour_height.nc'.format(
sector, year, month)
ncfile = dataset.addfile(out_fn, 'c', largefile=True)
ncfile.nc_define(dims, gattrs, dimvars)
data = np.zeros((tdim.length, z, ydim.length, xdim.length))
dd = np.zeros((tdim.length, z, ydim.length, xdim.length))
#read, merge and output
if sector in sectors_al:
print('Allocating: {}'.format(sector))
else:
print('Do not need to be allocated: {}'.format(sector))
print('Write data to file...')
for var in f.varnames():
if var == 'lat' or var == 'lon':
continue
else:
print(var)
dd[:, 0, :, :] = f[var][:]
if sector in sectors_al:
if sector == 'energy':
data[:, 1, :, :] = dd[:, 0, :, :] * 0.1
data[:, 2, :, :] = dd[:, 0, :, :] * 0.1
data[:, 3, :, :] = dd[:, 0, :, :] * 0.3
data[:, 4, :, :] = dd[:, 0, :, :] * 0.2
data[:, 5, :, :] = dd[:, 0, :, :] * 0.2
data[:, 6, :, :] = dd[:, 0, :, :] * 0.1
if sector == 'industry':
data[:, 0, :, :] = dd[:, 0, :, :] * 0.5
data[:, 1, :, :] = dd[:, 0, :, :] * 0.3
data[:, 2, :, :] = dd[:, 0, :, :] * 0.2
else:
data[:, 0, :, :] = dd[:, 0, :, :]
#Turn nan to zero
data[data == np.nan] = 0
###test###
'''
if sector == 'energy':
data[:, 0, :, :] = 0
'''
###test###
ncfile.write(var, data)
ncfile.close()
f.close()
else:
print('File not exist: {}'.format(fn))
continue
print('Allocate of height finished!')
