def do_mask_nan_utc(fname, key):
read_name = os.path.join(path_goci_aod, fname)
write_name = os.path.join(path_goci_filter, fname)
arr = matlab.loadmat(read_name)[key]
arr[nan_utc] = np.nan
matlab.savemat(fname=write_name, data={key: arr})
def create_grid_goci_surrPx(grid_goci):
DT = matlab.delaunayTriangulation(grid_goci)
ti = DT.find_simplex(grid_kor)
triPx = DT.simplices[ti, :]
triPx_UL = np.min(triPx, axis=1)
I, J = matlab.ind2sub(lon_goci.shape, triPx_UL)
surrPx = np.zeros((len(I), 4))
surrPx[:, 0] = matlab.sub2ind(lon_goci.shape, I, J) # Upper Left
surrPx[:, 1] = matlab.sub2ind(lon_goci.shape, I, J + 1) # Upper Right
surrPx[:, 2] = matlab.sub2ind(lon_goci.shape, I + 1, J) # Lower Left
surrPx[:, 3] = matlab.sub2ind(lon_goci.shape, I + 1, J + 1) # Lower Right
lon_surr = lon_goci.T.flatten()[list(
surrPx.flatten().astype(int))].reshape(
surrPx.shape) # lon_goci[surrPx]
lat_surr = lat_goci.T.flatten()[list(
surrPx.flatten().astype(int))].reshape(surrPx.shape)
diff_lon = lon_surr - np.tile(grid_kor[:, 0][:, None], (1, 4))
diff_lat = lat_surr - np.tile(grid_kor[:, 1][:, None], (1, 4))
d_surr = np.sqrt(np.power(diff_lon, 2) + np.power(diff_lat, 2))
invDsq = 1. / np.power(d_surr, 2)
k = np.zeros((len(I), 1))
for i in range(len(I)):
k[i] = surrPx[i, d_surr[i, :] == min(d_surr[i, :])]
fname = os.path.join(path_processed_grid, 'grid_goci_surrPx.mat')
matlab.savemat(fname, {
'surrPx': surrPx,
'd_surr': d_surr,
'invDsq': invDsq,
'k': k
})
if weight_sum is None:
weight_sum = weight
else:
weight_sum = np.vstack([weight_sum, weight])
min_dist = np.min(stn_GOCI6km_temp[:, 13])
stn_GOCI6km_temp2 = stn_GOCI6km_temp[
stn_GOCI6km_temp[:, 13] == min_dist, :]
if stn_GOCI6km_temp2.shape[0] != 1:
stn_GOCI6km_temp2 = stn_GOCI6km_temp2[
0, :].reshape(1, -1)
weight_sum = np.sum(weight_sum, axis=0)
stn_GOCI6km_temp2[:, 5:11] = np.divide(
np.nansum(stn_GOCI6km_temp[:, 5:11], axis=0),
weight_sum)
stn_GOCI6km = np.vstack(
[stn_GOCI6km, stn_GOCI6km_temp2[:, :-1]])
stn_GOCI6km = stn_GOCI6km[
stn_GOCI6km[:, 12].argsort()] # sort by scode2
if stn_GOCI6km_yr is None:
stn_GOCI6km_yr = stn_GOCI6km
else:
stn_GOCI6km_yr = np.vstack([stn_GOCI6km_yr, stn_GOCI6km])
print(doy)
jp_stn_GOCI6km_yr = stn_GOCI6km_yr
matlab.savemat(os.path.join(path_stn_jp, fname_save),
{'jp_stn_GOCI6km_yr': jp_stn_GOCI6km_yr})
tElapsed = time.time() - tStart
else:
mm = 12
for k in range(1,mm+1):
ndata_temp2 = ndata_temp[ndata_temp[:,2]==k,:]
if len(ndata_temp2)!=0:
yrmon = yr*100+k
idx = (stn_info_cn[j,4]<=yrmon)&(stn_info_cn[j,5]>=yrmon)
if idx: # True
ndata_temp2[:,-1]=stn_info_cn[j,1]
if ndata_scode is None:
ndata_scode = ndata_temp2
else:
ndata_scode= np.vstack([ndata_scode,ndata_temp2])
if ((j+1)%100)==0:
fname = f'stn_scode_data_{yr}_{j+1-99:04d}.mat'
matlab.savemat(os.path.join(path_station,'Station_CN', fname), {'ndata_scode':ndata_scode})
ndata_scode = None
elif (j+1)==stn_info_cn.shape[0]:
if ndata_scode is not None:
fname = f'stn_scode_data_{yr}_{np.int((j+1)/100)*100+1:04d}.mat'
matlab.savemat(os.path.join(path_station,'Station_CN', fname), {'ndata_scode':ndata_scode})
print (f'{j+1} / {stn_info_cn.shape[0]}')
ndata_scode = None
file_list = glob.glob(os.path.join(path_station,'Station_CN', f'stn_scode_data_{yr}_*.mat'))
file_list.sort()
for fname in file_list:
print (f'Reading ... {os.path.basename(fname)}')
ndata_scode_temp = matlab.loadmat(fname)['ndata_scode']
print (ndata_scode_temp.shape)
if ndata_scode is None:
YEARS = [2016]
for yr in YEARS:
flist = glob.glob(
os.path.join(path_myd_processed, '03mask', str(yr), "*.tif"))
flist.sort()
for fname in flist:
tStart = time.time()
print(os.path.basename(fname))
with rio.open(fname) as src:
ndvi = src.read(1)
ndvi = np.float64(ndvi)
ndvi[ndvi <= -32767] = np.nan
ndvi = np.divide(ndvi, 10000)
ndvi[np.abs(ndvi) >= 0.99] = np.nan
values = ndvi.ravel(order='F')
ndivi = griddata(points=points,
values=values,
xi=(lon_goci, lat_goci),
method='linear')
matlab.savemat(
os.path.join(
path_ea_goci_ndivi, str(yr),
f'EA_MODIS_NDVI_{os.path.basename(fname)[10:-4]}.mat'),
{'ndvi': ndivi})
del ndvi
print(f'EA_MODIS_NDVI_{os.path.basename(fname)[10:-4]}.mat')
tElapsed = time.time() - tStart
print(f'time taken : {tElapsed}')
print(yr)
'float64')
GOCI_ndvi = matlab.hdfread(
fname, 'NDVI_from_TOA_Reflectance_660_865nm').astype('float64')
GOCI_dai = matlab.hdfread(
fname, 'Dust_Aerosol_Index_from_412_443nm').astype('float64')
GOCI_aod[(GOCI_aod < -0.05) | (GOCI_aod > 3.6)] = np.nan
GOCI_fmf[(GOCI_fmf < 0) | (GOCI_fmf > 1)] = np.nan
GOCI_ssa[(GOCI_ssa < 0) | (GOCI_ssa > 1)] = np.nan
GOCI_ae[(GOCI_ae < 0) | (GOCI_ae > 3)] = np.nan
GOCI_type[(GOCI_type < 0) | (GOCI_type > 6)] = np.nan
GOCI_ndvi[(GOCI_ndvi < -1) | (GOCI_ndvi > 1)] = np.nan
fname_temp = f'{yr}_{doy:03d}_{utc:02d}.mat'
matlab.savemat(fname=os.path.join(path_goci_aod, 'AOD', str(yr),
f'GOCI_AOD_{fname_temp}'),
data={'GOCI_aod': GOCI_aod})
matlab.savemat(fname=os.path.join(path_goci_aod, 'FMF', str(yr),
f'GOCI_FMF_{fname_temp}'),
data={'GOCI_fmf': GOCI_fmf})
matlab.savemat(fname=os.path.join(path_goci_aod, 'SSA', str(yr),
f'GOCI_SSA_{fname_temp}'),
data={'GOCI_ssa': GOCI_ssa})
matlab.savemat(fname=os.path.join(path_goci_aod, 'AE', str(yr),
f'GOCI_AE_{fname_temp}'),
data={'GOCI_ae': GOCI_ae})
matlab.savemat(fname=os.path.join(path_goci_aod, 'Type', str(yr),
f'GOCI_Type_{fname_temp}'),
data={'GOCI_type': GOCI_type})
matlab.savemat(fname=os.path.join(
path_goci_aod,
data = matlab.h5read(
read_fname,
'/HDFEOS/GRIDS/ColumnAmountNO2/Data Fields/ColumnAmountNO2TropCloudScreened'
)
data = np.float64(data.T)
# 720X1440
data[
data <=
-1.2676506e+30] = np.nan # Assign NaN value to pixel that is out of valid range
data = data * 3.7216e-17
data_yr[:, doy - 1] = data.ravel(order='F')
out_fname = os.path.join(path_write, f'OMNO2d_trop_CS_{yr}_DU.mat')
matlab.check_make_dir(os.path.dirname(out_fname))
data_yr[np.isnan(data_yr)] = -9999
matlab.savemat(out_fname, {'data_yr': data_yr})
tElapsed = time.time() - tStart
print(f'{tElapsed} second')
del data, data_yr
print('==========================================================')
### OMSO2e
print('OMSO2e')
for yr in YEARS:
tStart = time.time()
doy_000 = matlab.datenum(f'{yr}0000')
file_list = glob.glob(
os.path.join(path_read, 'L3_grid', 'OMSO2e', str(yr), '*.he5'))
file_list.sort()
if yr % 4 == 0: days = 366
if i == days:
data24 = matlab.loadmat(
os.path.join(rdaps_path, str(yr + 1),
f'RDAPS_{yr+1}_001_00.mat')) #rdaps
else:
data24 = matlab.loadmat(
os.path.join(curr_path,
f'RDAPS_{yr}_{i+1:03d}_00.mat')) # rdaps
data24 = data24['rdaps']
for j in range(1, 5 + 1):
rdaps = np.multiply((data06 - data00),
(j / 6)) + data00 # 01 to 05 UTC
fname = f'RDAPS_{yr}_{i:03d}_{j:02d}.mat'
matlab.savemat(curr_path, fname, {'rdaps': rdaps})
rdaps = np.multiply((data12 - data06),
(j / 6)) + data06 # 07 to 11 UTC
fname = f'RDAPS_{yr}_{i:03d}_{j+6:02d}.mat'
matlab.savemat(curr_path, fname, {'rdaps': rdaps})
rdaps = np.multiply((data18 - data12),
(j / 6)) + data12 # 13 to 17 UTC
fname = f'RDAPS_{yr}_{i:03d}_{j+12:02d}.mat'
matlab.savemat(curr_path, fname, {'rdaps': rdaps})
rdaps = np.multiply((data24 - data18),
(j / 6)) + data18 # 19 to 23 UTC
fname = f'RDAPS_{yr}_{i:03d}_{j+18:02d}.mat'
matlab.savemat(curr_path, fname, {'rdaps': rdaps})
doy000 = matlab.datenum(f'{yr}00000')
date_list = dict()
for x in range(1, days + 1):
date_list[x] = matlab.datestr(doy000 + x)
dates = df['doy'].apply(lambda x: date_list[x])
dates = pd.DatetimeIndex(dates)
df['yr'] = dates.year
df['mon'] = dates.month
df['day'] = dates.day
df.reset_index(drop=True, inplace=True)
for col in cols:
df.loc[df[col] >= 9997, col] = np.nan
df.loc[df[col] == -9999, col] = np.nan
df.dropna(axis=0, subset=cols, how='all',
inplace=True) # axis=0: row, subset: 기준
ndata = df[header_ndata].values
ind = np.lexsort(
(ndata[:, 4], ndata[:, 0], ndata[:,
11])) # sort 11->0->4. 0 is primary
ndata = ndata[ind]
matlab.savemat(
os.path.join(path_stn_jp, 'stn_code_data', f'stn_code_data_{yr}.mat'),
{'ndata': ndata})
with h5py.File(
os.path.join(path_stn_jp, 'stn_code_data',
f'stn_code_data_{yr}.mat'), 'a') as dst:
dst['header_ndata'] = header_ndata
t2 = time.time() - t1
print(f'time taken : {t2:.3f}')
data2 = copy.deepcopy(temp.data)
data2 = np.transpose(data2, (3, 2, 1, 0))
data2 = np.rot90(data2)
emiss_all[:, :, j - 1, :2] = np.float64(
np.squeeze(data2[:, :, 0, 1:3])) # forecast 01-02UTC
except:
pass
for k in range(24):
utc = k + 1
emiss = emiss_all[:, :, :, k]
if k == 23: # last
if doy == days: # last
fname = f'emiss_27km_{yr+1}_001_00.mat'
matlab.savemat(
os.path.join(write_path, KNU_dir, str(yr + 1), fname),
{'emiss': emiss})
else:
fname = f'emiss_27km_{yr}_{doy+1:03d}_00.mat'
matlab.savemat(
os.path.join(write_path, KNU_dir, str(yr), fname),
{'emiss': emiss})
else:
fname = f'emiss_27km_{yr}_{doy:03d}_{utc:02d}.mat'
matlab.savemat(
os.path.join(write_path, KNU_dir, str(yr), fname),
{'emiss': emiss})
print(f'{yr}_{doy:03d}')
#except:
# print (f'{yr}_{doy:03d}_no_data!!!!')
# pass
# 고농도 지우기
if len(high) != 0:
idx_high = np.where(
data_stn[:, 25] <= high[-1, 25]
& data_stn[:, 23] >= 150 & data_stn[:, -1] == 1)[0]
data_stn = data_stn[~idx_high, :]
elif high_rate > 30 and low_rate > 30:
print(f'high_rate = {high_rate:3.2f} \n')
print(f'low_rate = {low_rate:3.2f} \n')
# 고농도 지우기
idx_high = np.where(
data_stn[:, 25] <= high[-1, 25]
& data_stn[:, 23] >= 150 & data_stn[:, -1] == 1)[0]
ddata_stn = data_stn[~idx_high, :]
# 저농도 지우기
idx_low = np.where(
data_stn[:, 25] <= low[-1, 25] & data_stn[:, 23] <= 60
& data_stn[:, -1] == 1)
data_stn = data_stn[~idx_low, :]
# print (('data_stn 파일 갯수 줄이기.\n')
# sample수 줄이는 코드 추가필요 전체적인 갯수줄이기
# data_stn[:,-1]==1 인 날에대해서 줄이기
print(utc)
fname = f'{target[i]}_RTT_{yr}_{doy:03}_{utc:02d}.csv'
tmp_df = pd.DataFrame(data_stn, columns=header2)
matlab.savemat(os.path.join(path_rtt, 'dataset/', target[i],
'/new/', fname),
float_format='%7.7f')
print(doy)
print(yr)
lat_gpm, lon_gpm = mat['lat_gpm'], mat['lon_gpm']
del mat
lat_gpm = lat_gpm[1100:1400, 2900:3300] # N50 W110 S20 E150
lon_gpm = lon_gpm[1100:1400, 2900:3300]
points = np.array([lon_gpm.ravel(order='F'), lat_gpm.ravel(order='F')]).T
del lon_gpm, lat_gpm
print(f'points shape : {points.shape}')
YEARS = [2016]
for yr in YEARS:
flist = glob.glob(os.path.join(path_gpm_processed, str(yr), '*.mat'))
flist.sort()
for fname in flist:
tStart = time.time()
precip = matlab.loadmat(fname)['precip']
precip = precip[1100:1400, 2900:3300]
values = precip.ravel(order='F')
precip = griddata(points=points,
values=values,
xi=(lon_goci, lat_goci),
method='linear')
matlab.savemat(
os.path.join(path_ea_goci_gpm, str(yr),
f'EA6km_{os.path.basename(fname)}'),
{'precip': precip})
del precip
print(os.path.basename(fname))
tElapsed = time.time() - tStart
print(f'time taken : {tElapsed}')
print(yr)
OX[nanidx[:, 2] == 1, :] = np.nan
NO2[nanidx[:, 3] == 1, :] = np.nan
PM10[nanidx[:, 4] == 1, :] = np.nan
PM25[nanidx[:, 5] == 1, :] = np.nan
allvar = np.vstack([
SO2.ravel(order='F'),
CO.ravel(order='F'),
OX.ravel(order='F'),
NO2.ravel(order='F'),
PM10.ravel(order='F'),
PM25.ravel(order='F')
]).T
nanidx_allvar = np.sum(np.isnan(allvar), axis=1) == 6
ndata_temp[:, 5:11] = allvar
ndata_temp = ndata_temp[~(nanidx_allvar == 1)]
if stn_JP is None:
stn_JP = ndata_temp
else:
stn_JP = np.vstack([stn_JP, ndata_temp])
tElapsed_doy = time.time() - tStart_doy
print(f'{yr}_{doy} -- {tElapsed_doy:3.4f} sec')
else:
print(f'Less than 4 hourly data in {doy:03d} (DOY) \n')
matlab.savemat(
os.path.join(path_stn_jp, 'stn_code_data',
f'stn_code_data_rm_outlier_{yr}.mat'), {'stn_JP': stn_JP})
print(yr)
lon_goci, lat_goci = mat['lon_goci'], mat['lat_goci']
del mat
coords = ((lon_goci - xmin) / dx, (lat_goci - ymin) / dy)
# SRTM DEM
with rio.open(os.path.join(path_grid_raw, 'SRTM_DEM_1km.tif')) as src:
dem = src.read(1)
dem = dem.astype('float64')
dem[dem == src.nodata] = np.nan
dem[dem < 0] = np.nan
tStart = time.time()
dem = map_coordinates(dem[::-1].T, coords, order=1)
#dem = griddata(points, values, (lon_goci, lat_goci), method='linear')
tElapsed = time.time() - tStart
print(f'time taken : {tElapsed}')
matlab.savemat(os.path.join(path_ea_goci, 'stationary', 'EA6km_SRTM_DEM.mat'),
{'dem': dem})
print('stationary/EA6km_SRTM_DEM.mat')
# Population density
mat = matlab.loadmat(
os.path.join(path_grid_raw, 'grid_EA_30sec_GPWv4_PopDens.mat'))
lon_EA_30sec, lat_EA_30sec = mat['lon_EA_30sec'], mat['lat_EA_30sec']
del mat
dx = 0.00833333
dy = 0.00833333
xmin = np.min(lon_EA_30sec[0])
ymin = np.min(lat_EA_30sec[:, 0])
coords = ((lon_goci - xmin) / dx, (lat_goci - ymin) / dy)
popDens_all = np.zeros((3600, 4800, 21))
for yr in range(2000, 2020 + 1, 5):
dtype=h5py.string_dtype(encoding='utf-8'))
## stn_scode_data for South Korea
YEARS = [2016]
for yr in YEARS:
if yr==2019:
ndata=matlab.loadmat(os.path.join(path_stn_kor,'stn_code_data', 'stn_code_data_2019_010100_042300.mat'))['ndata']
else:
ndata=matlab.loadmat(os.path.join(path_stn_kor,'stn_code_data', f'stn_code_data_{yr}.mat'))['ndata']
ndata = np.hstack([ndata, np.zeros([ndata.shape[0], 1])]) # add column for scode2
ndata[ndata<0] = np.nan
ndata_scode = None
# Assign scode2
for j in range(stn_info.shape[0]):
ndata_temp = ndata[ndata[:,11]==stn_info[j,0],:]
for k in range(1,12+1):
ndata_temp2 = ndata_temp[ndata_temp[:,2]==k,:]
if len(ndata_temp2)!=0:
yrmon = yr*100+k
idx = (stn_info[j,4]<=yrmon)&(stn_info[j,5]>yrmon)
if idx:
ndata_temp2[:,12]=stn_info[j,1]
if ndata_scode is None:
ndata_scode = ndata_temp2
else:
ndata_scode=np.vstack([ndata_scode,ndata_temp2])
print (f'{j} / {stn_info.shape[0]}')
fname = f'stn_scode_data_{yr}.mat'
matlab.savemat(os.path.join(path_stn_kor, 'stn_scode_data', fname), {'ndata_scode':ndata_scode})
with h5py.File(os.path.join(path_stn_kor, 'stn_scode_data', fname), 'a') as dst:
dst['header_ndata'] = header_ndata
stn[:, 0] = doy
stn[:, 1] = yr
stn[:, 2] = mm
stn[:, 3] = dd
stn[:, 4] = CST
stn[:, 20] = scode_unq
ttidx = np.sum(stn_hour == CST)
#print (ttidx)
if ttidx == 15:
stn[:, 5:20] = stn_value[stn_hour == CST].T
else:
#nan, without time
print(f'NO data in {CST:2.0f} (Local Time) on {doy:03d}')
if stn_doy is None:
stn_doy = stn
else:
stn_doy = np.concatenate((stn_doy, stn), axis=0)
else:
stn_doy = np.full((len(scode_unq) * 24, 21), np.nan)
print(f'NO file in {doy:03d} (DOY)')
if stn_yr is None:
stn_yr = stn_doy
else:
stn_yr = np.concatenate((stn_yr, stn_doy), axis=0)
fname = f'stn_code_data_{yr}.mat'
matlab.savemat(
os.path.join(path_station, 'Station_CN', 'stn_code_data', fname),
{'stn_yr': stn_yr})
print(yr)
matlab.loadmat(os.path.join(path_stn_cn, 'Station_CN',
fname)) # cn_stn_GOCI6km_yr
cn_stn_GOCI6km_yr = mat['cn_stn_GOCI6km_yr']
del mat
stn_GOCI6km_yr[:, 4] = stn_GOCI6km_yr[:, 4] - 9
cn_stn_GOCI6km_yr[:, 4] = cn_stn_GOCI6km_yr[:, 4] - 8
stn = np.concatenate(
(stn_GOCI6km_yr,
cn_stn_GOCI6km_yr[:, [0, 1, 2, 3, 4, 10, 18, 14, 12, 8, 6, 20, 21]]),
axis=0)
del stn_GOCI6km_location, dup_scode2_GOCI6km, header_stn_GOCI6km_location
del cn_stn_GOCI6km_location, cn_dup_scode2_GOCI6km, header_cn_stn_GOCI6km_location
del stn_GOCI6km_yr, cn_stn_GOCI6km_yr
stn_nanidx = []
for doy in range(1, nanidx.shape[1] + 1):
for utc in range(7 + 1):
nanidx_temp = nanidx[:, doy - 1, utc]
stn_temp = stn[stn[:, 0] == doy & stn[:, 4] == utc, :]
for k in range(stn_temp.shape[0]):
pid = stn_location[stn_location[:, 1] == stn_temp[k, 12], 4]
stn_temp[k, 13] = nanidx_temp[pid]
stn_nanidx = np.concatenate((stn_nanidx, stn_temp), axis=0)
stn_fill = stn_nanidx[stn_nanidx[:, 13] == 0, :13]
fname = f'stn_rf_day_{yr}.mat'
matlab.savemat(os.path.join(path_ea_goci, fname), {'stn_fill': stn_fill})
print(yr)
if data_new is None: data_new = data_temp
else: data_new = np.vstack([data_new, data_temp])
data_new[data_new[:, 7] >= 9997, 7] = np.nan
if yr % 4 == 0:
idx = (data_new[:, 2] == 2) & (data_new[:, 3] > 29)
data_new = data_new[~idx]
else:
idx = (data_new[:, 2] == 2) & (data_new[:, 3] > 28)
data_new = data_new[~idx]
tmp_idx = (data_new[:, 2] == 4) & (data_new[:, 3] == 31)
data_new = data_new[~tmp_idx]
tmp_idx = (data_new[:, 2] == 6) & (data_new[:, 3] == 31)
data_new = data_new[~tmp_idx]
tmp_idx = (data_new[:, 2] == 9) & (data_new[:, 3] == 31)
data_new = data_new[~tmp_idx]
tmp_idx = (data_new[:, 2] == 11) & (data_new[:, 3] == 31)
data_new = data_new[~tmp_idx]
stn_tbl = pd.DataFrame(data_new, columns=header_p)
stn_tbl = stn_tbl.apply(pd.to_numeric)
matlab.savemat(
os.path.join(path_stn_jp, 'byPollutant',
f'JP_stn{varname}_{yr}.mat'),
{col: stn_tbl[col].values
for col in stn_tbl.columns})
with h5py.File(
os.path.join(path_stn_jp, 'byPollutant',
f'JP_stn{varname}_{yr}.mat'), 'a') as dst:
dst['header'] = np.array(header_p,
dtype=h5py.string_dtype(encoding='utf-8'))
#stn_tbl.to_csv(os.path.join(path_stn_jp, f'JP_stn{varname}_{yr}.csv'),sep=',',na_rep='NaN')
mat = matlab.loadmat(
os.path.join(path_korea_cases, f'nanidx_1km_hourly_{yr}')) # nanidx
nanidx = mat['nanidx']
del mat
# station
mat = matlab.loadmat(
os.path.join(path_stn_kr,
f'Station_1km_rm_outlier_{yr}_weight.mat')) # stn_1km_yr
stn_1km_yr = mat['stn_1km_yr']
del mat
stn_1km_yr[:, 4] = stn_1km_yr[:, 4] - 9 # KST to UTC
stn = stn_1km_yr
stn_nanidx = []
for doy in range(1, nanidx.shape[1]):
for utc in range(7 + 1):
nanidx_temp = nanidx[:, doy - 1, utc]
stn_temp = stn[stn[:, 0] == doy & stn[:, 4] == utc, :]
for k in range(stn_temp.shape[0]):
pid = stn_location[stn_location[:, 1] == stn_temp[k, 12], 4]
stn_temp[k, 13] = nanidx_temp[pid]
stn_nanidx = np.concatenate((stn_nanidx, stn_temp), axis=0)
stn_fill = stn_nanidx[stn_nanidx[:, 13] == 0, :13]
fname = f'KR_1km_stn_rf_day_{yr}.mat'
matlab.savemat(os.path.join(path_korea_cases, fname),
{'stn_fill': stn_fill})
print(yr)
stn_GOCI6km_temp[k, 5:11], weight)
if weight_sum is None:
weight_sum = weight
else:
weight_sum = np.vstack([weight_sum, weight])
min_dist = np.min(stn_GOCI6km_temp[:, 13])
stn_GOCI6km_temp2 = stn_GOCI6km_temp[
stn_GOCI6km_temp[:, 13] == min_dist, :]
if stn_GOCI6km_temp2.shape[0] != 1:
stn_GOCI6km_temp2 = stn_GOCI6km_temp2[0, :]
# 픽셀중심에 더 가까운 관측소의 scode2를 사용하기 위함. 관측값은 가중평균한 값으로 다시 할당될거이므로 신경 쓰지말기
weight_sum = np.sum(weight_sum, axis=0)
stn_GOCI6km_temp2[0, 5:11] = np.divide(
np.nansum(stn_GOCI6km_temp[:, 5:11], axis=0),
weight_sum)
stn_GOCI6km = np.vstack(
[stn_GOCI6km, stn_GOCI6km_temp2[:, :-1]])
stn_GOCI6km = stn_GOCI6km[
stn_GOCI6km[:, 12].argsort()] # sort by scode2
if stn_GOCI6km_yr is None:
stn_GOCI6km_yr = stn_GOCI6km
else:
stn_GOCI6km_yr = np.vstack([stn_GOCI6km_yr, stn_GOCI6km])
print(doy)
matlab.savemat(
os.path.join(path_stn_kor, f'Station_GOCI6km_{yr}_weight.mat'),
{'stn_GOCI6km_yr': stn_GOCI6km_yr})
tElapsed = time.time() - tStart
print(f'time taken : {tElapsed}')
YEARS = [2016]
for yr in YEARS:
file_list = glob.glob(os.path.join(path_bess_raw, str(yr), '*.nc'))
file_list.sort()
for fname in file_list:
tStart = time.time()
print(f'Reading ... {os.path.basename(fname)}')
if yr <= 2016: # Until 2016 (nc)
ncfile = netCDF4.Dataset(fname)
bess = np.array(
ncfile.variables['surface_downwelling_shortwave_flux_in_air']
).T
ncfile.close()
else: # 2017 (mat)
bess = matlab.loadmat(fname)['bess']
bess = bess.astype('float64')
bess[bess == -9999] = np.nan
values = bess.ravel(order='F')
tStart = time.time()
RSDN = griddata(points,
values,
xi=(lon_goci, lat_goci),
method='linear')
write_fname = f'EA6km_BESS_RSDN_{yr}_{os.path.basename(fname)[-6:-3]}.mat'
matlab.savemat(os.path.join(path_ea_goci_bess, str(yr), write_fname),
{'RSDN': RSDN})
del bess, RSDN
print(write_fname)
tElapsed = time.time() - tStart
print(f'time taken : {tElapsed}')
print(yr)
# nan masking to prediction PM concentration.
pred[nanidx==1,:] = np.nan
if yr==2015 & doy==30:
daily[0:218999,0:7] = np.nan
daily[:,utc] = pred
else:
daily[:,utc] = pred
print (utc)
#close all
except:
print (f"{doy:03d}_{utc:02d}")
# save hourly prediction matrix
fname = f"{target[i]}_RTT_EA6km_{yr}_{doy:03d}"
matlab.savemat(os.path.join(path_ea_goci, 'RF_map/',type_list[t],target[i],'daily/',fname), {'daily':daily})
#clear pred
daily_avg = np.nanmean(daily,axis=1)
if yr==2015:
if len(annual)==0:
annual = np.zeros((218999, 29))*np.nan
annual[:,doy] = daily_avg
else:
annual[:,doy] = daily_avg
# reshaping and mapping
daily_avg = daily_avg.reshape(lon_goci.shape)
fig, axs = matlab.m_kc(lon_goci,lat_goci,daily_avg)
"""
dup_scode2_GOCI6km = np.zeros((len(idx), np.max(unq_cnt) + 1))
dup_scode2_GOCI6km[:, 0] = idx
for i in range(len(idx)):
aa = stn[stn[:, 4] == dup_scode2_GOCI6km[i, 0], 1]
dup_scode2_GOCI6km[i, 1:len(aa) + 1] = aa
stn_GOCI6km_temp = stn[stn[:, 4] == idx[i], :]
stn_GOCI6km_temp = np.hstack(
[stn_GOCI6km_temp,
np.ones([stn_GOCI6km_temp.shape[0], 1])])
stn_GOCI6km = np.vstack([stn_GOCI6km, stn_GOCI6km_temp])
stn_GOCI6km = stn_GOCI6km[stn_GOCI6km[:, 1].argsort()]
jp_stn_GOCI6km_location = stn_GOCI6km
jp_dup_scode2_GOCI6km = dup_scode2_GOCI6km
header_jp_stn_GOCI6km_location = np.array(
[
'scode1', 'scode2', 'lat_org', 'lon_org', 'pxid', 'lat_px', 'lon_px',
'dist', 'avgid'
],
dtype=h5py.string_dtype(encoding='utf-8'))
matlab.savemat(
os.path.join(path_stn_jp, 'jp_stn_GOCI6km_location_weight_v2017.mat'), {
'jp_stn_GOCI6km_location': jp_stn_GOCI6km_location,
'jp_dup_scode2_GOCI6km': jp_dup_scode2_GOCI6km
})
with h5py.File(
os.path.join(path_stn_jp, 'jp_stn_GOCI6km_location_weight_v2017.mat'),
'a') as dst:
dst['header_jp_stn_GOCI6km_location'] = header_jp_stn_GOCI6km_location
typeOfLevel='heightAboveGround')[0].values
rdaps[:, :, 11] = np.squeeze(data)
data = rdaps_data.select(parameterName='Total precipitation')[0].values
rdaps[:, :, 12] = np.squeeze(data)
data = rdaps_data.select(name='Frictional velocity',
typeOfLevel='heightAboveGround')[0].values
rdaps[:, :, 13] = np.squeeze(data)
data = rdaps_data.select(
name='Convective available potential energy')[0].values
rdaps[:, :, 14] = np.squeeze(data)
data = rdaps_data.select(name='Surface roughness',
typeOfLevel='surface')[0].values
rdaps[:, :, 15] = np.squeeze(data)
data = rdaps_data.select(name='Latent heat net flux',
typeOfLevel='surface')[0].values
rdaps[:, :, 16] = np.squeeze(data)
data = rdaps_data.select(name='Specific humidity',
typeOfLevel='heightAboveGround')[0].values
rdaps[:, :, 17] = np.squeeze(data)
doy = matlab.datenum(os.path.basename(fname)[21:29]) - doy_000
utc = int(os.path.basename(fname)[29:31])
#fname = f'RDAPS_{yr}_{doy:03d}_{utc:02d}_006.mat'
fname = f'RDAPS_{yr}_{doy:03d}_{utc:02d}.mat'
matlab.savemat(os.path.join(path_rdaps_processed, str(yr), fname),
{'rdaps': rdaps})
print(fname)
tElapsed = time.time() - tStart
print(f'{tElapsed} second')
print(yr)
dup_scode2_GOCI6km[:, 0] = idx
for i in range(len(idx)):
aa = stn[stn[:, 4] == dup_scode2_GOCI6km[i, 0], 1]
dup_scode2_GOCI6km[i, 1:len(aa) + 1] = aa
stn_GOCI6km_temp = stn[stn[:, 4] == idx[i], :]
stn_GOCI6km_temp = np.hstack(
[stn_GOCI6km_temp,
np.ones([stn_GOCI6km_temp.shape[0], 1])])
stn_GOCI6km = np.vstack([stn_GOCI6km, stn_GOCI6km_temp])
stn_GOCI6km = stn_GOCI6km[stn_GOCI6km[:, 1].argsort()]
cn_stn_GOCI6km_location = stn_GOCI6km
cn_dup_scode2_GOCI6km = dup_scode2_GOCI6km
header_cn_stn_GOCI6km_location = np.array(
[
'scode1', 'scode2', 'lat_org', 'lon_org', 'pxid', 'lat_px', 'lon_px',
'avgid', 'dist'
],
dtype=h5py.string_dtype(encoding='utf-8'))
matlab.savemat(
os.path.join(path_station,
'Station_CN/cn_stn_GOCI6km_location_weight.mat'), {
'cn_stn_GOCI6km_location': cn_stn_GOCI6km_location,
'cn_dup_scode2_GOCI6km': cn_dup_scode2_GOCI6km
}),
with h5py.File(
os.path.join(path_station, 'Station_CN',
'cn_stn_GOCI6km_location_weight.mat'), 'a') as dst:
dst['header_cn_stn_GOCI6km_location'] = header_cn_stn_GOCI6km_location
if yr%4==0: days = 366
else: days = 365
for doy in range(1, days+1): # for doy=294:300 %1:days
for utc in range(23+1):
tStart = time.time()
print (os.path.join(path_read, str(yr), f'emiss_27km_{yr}_{doy:03d}_{utc:02d}.mat'))
try:
emiss = matlab.loadmat(os.path.join(path_read, str(yr), f'emiss_27km_{yr}_{doy:03d}_{utc:02d}.mat'))['emiss']
EA_emis = np.zeros((nr_goci,nc_goci,14))
EA_emis[:,:,0]=griddata(points, emiss[:,:,0].ravel(order='F'),(lon_goci, lat_goci),method='linear') # ISOPRENE
EA_emis[:,:,1]=griddata(points,emiss[:,:,1].ravel(order='F'),(lon_goci, lat_goci),method='linear') # TRP1
EA_emis[:,:,2]=griddata(points,emiss[:,:,4].ravel(order='F'),(lon_goci, lat_goci),method='linear') # CH4
EA_emis[:,:,3]=griddata(points,emiss[:,:,5].ravel(order='F'),(lon_goci, lat_goci),method='linear') # NO
EA_emis[:,:,4]=griddata(points,emiss[:,:,6].ravel(order='F'),(lon_goci, lat_goci),method='linear') # NO2
EA_emis[:,:,6]=griddata(points,emiss[:,:,7].ravel(order='F'),(lon_goci, lat_goci),method='linear') # NH3
EA_emis[:,:,6]=griddata(points,emiss[:,:,9].ravel(order='F'),(lon_goci, lat_goci),method='linear') # HCOOH
EA_emis[:,:,7]=griddata(points,emiss[:,:,10].ravel(order='F'),(lon_goci, lat_goci),method='linear') # HCHO
EA_emis[:,:,8]=griddata(points,emiss[:,:,15].ravel(order='F'),(lon_goci, lat_goci),method='linear') # CO
EA_emis[:,:,9]=griddata(points,emiss[:,:,36].ravel(order='F'),(lon_goci, lat_goci),method='linear') # SO2
EA_emis[:,:,10]=griddata(points,emiss[:,:,41].ravel(order='F'),(lon_goci, lat_goci),method='linear') # PMFINE
EA_emis[:,:,11]=griddata(points,emiss[:,:,42].ravel(order='F'),(lon_goci, lat_goci),method='linear') # PNO3
EA_emis[:,:,12]=griddata(points,emiss[:,:,43].ravel(order='F'),(lon_goci, lat_goci),method='linear') # POA
EA_emis[:,:,13]=griddata(points,emiss[:,:,44].ravel(order='F'),(lon_goci, lat_goci),method='linear') # PSO4
except IOError:
print ('No data and make nan matrix file')
EA_emis = np.full((nr_goci,nc_goci, 14), np.nan)
pass
matlab.savemat(os.path.join(path_ea_goci_emis, str(yr), f'EA6km_EMIS_{yr}_{doy:03d}_{utc:02d}.mat'), {'EA_emis':EA_emis})
print (f'EA6km_EMIS_{yr}_{doy:03d}_{utc:02d}')
tElapsed = time.time() - tStart
print (f'{tElapsed} second')
# points를 아래와 같이 배치
# (x=0,y=0), (x=0,y=1), ..., (x=0, y=n)
# (x=1,y=0), (x=1,y=1), ...
# (x=2,y=0), (x=2,y=1), ...
class_name = [
'barren', 'crop', 'forest', 'grass', 'savannas', 'shrub', 'snow', 'urban',
'water', 'wetland'
]
YEARS = [2016]
for yr in YEARS:
for col in class_name:
tStart = time.time()
with rio.open(
os.path.join(path_mcd_processed, '03_LC_ratio', str(yr),
f'EA_{col}_ratio_r6_500m_{yr}.tif')) as src:
LC_ratio = src.read(1)
LC_ratio = np.float64(LC_ratio)
LC_ratio[LC_ratio == src.nodata] = np.nan
LC_ratio[LC_ratio < 0] = np.nan #% nodata value -3.40282346639e+038
LC_ratio = LC_ratio[::-1].T
arr = map_coordinates(LC_ratio[::-1].T, coords, order=1)
matlab.savemat(
os.path.join(path_ea_goci_modis, f'EA6km_{col}_ratio_r6_{yr}.mat'),
{f'LC_{col}': arr})
print(f'EA6km_{col}_ratio_r6_{yr}')
del LC_ratio, arr
tElapsed = time.time() - tStart
print(f'time taken : {tElapsed}')
size = (1800, 3600, 48)
gpm = np.zeros(size)
doy = matlab.datenum(os.path.basename(list_temp[0])[21:29]) - doy0 + 1
print(f'doy: {doy}')
for j, fname in enumerate(list_temp[:48]):
gpm_temp = matlab.h5read(fname, '/Grid/precipitationCal')
gpm_temp = np.float64(gpm_temp)
gpm_temp[gpm_temp < -9999] = np.nan
gpm[:, :, j] += gpm_temp
precip = np.nansum(gpm, axis=2)
precip *= 0.5
ap_fname = os.path.join(path_gpm_processed, str(yr),
f'gpm_AP_{yr}_{doy:03d}_UTC00.mat')
print(f'Saving ... {ap_fname}')
matlab.savemat(ap_fname, {'precip': precip})
#for aa in range(2, len(list_gpm), 2):
for aa in range(2, len(list_gpm) - 48 + 1, 2):
tStart = time.time()
gpm[:, :, 0:46] = gpm[:, :, 2:]
gpm[:, :, -1] = 0
gpm[:, :, -2] = 0
#list_temp = list_gpm[aa:aa+2]
list_temp = list_gpm[aa + 45:aa + 48]
doy = matlab.datenum(os.path.basename(list_gpm[aa])[21:29]) - doy0 + 1
UTC = os.path.basename(list_gpm[aa])[31:33]
for j in range(2):
print(f'Reading ... {list_temp[j]}')
gpm_temp = matlab.h5read(list_temp[j], '/Grid/precipitationCal')
@njit(error_model='numpy')
def calculate(data):
data_conv = np.full(data.shape, np.nan)
for k in range(data.shape[0]):
for t in range(data.shape[1]):
t_ = t + 1
w_sum = 0
x_sum = 0
for n in range(data.shape[1]):
n_ = n + 1
if np.isnan(data[k, n]) == 0:
w = np.exp(
(-(t_ - n_)**2) / (2 * sigma**2)
) # calcuate weights using t,n(days) with sigma # (k+1)-(n-1)=k-n
x = data[k, n] * w
w_sum += w
x_sum += x
data_conv[k, t] = x_sum / w_sum
print(k)
return data_conv
data_conv = calculate(data)
matlab.savemat(
os.path.join(
path_read,
f'tempConv_{pname}_sigma{sigma}_{YEARS[0]}_{YEARS[len(YEARS)-1]}.mat'
), {
'data_conv': data_conv,
'data': data
})
data_datenum = [matlab.datenum(val) for val in dstr]
doy_000 = matlab.datenum(f'{yr}00000')
doy = np.array([val - doy_000 for val in data_datenum])
stn_value = stn_tbl_temp[:, 2:15]
data = np.hstack([
doy.reshape(-1, 1), dvec, stn_value[:,
[0, 1, 5, 6, 3, 10, 11]],
scode.reshape(-1, 1)
])
# {'doy','yr','mon','day','KST','SO2','CO','OX','NO2','PM10','PM25','scode'}
if stn_mm is None: stn_mm = data
else: stn_mm = np.vstack([stn_mm, data])
matlab.savemat(
os.path.join(path_stn_jp, 'stn_code_data',
f'stn_code_data_{yr}_{mm:02d}.mat'),
{'stn_mm': stn_mm})
print(mm)
print('Stack monthly data to yearly data')
stn_yr = None
for mm in range(1, 12 + 1):
stn_mm = matlab.loadmat(
os.path.join(path_stn_jp, 'stn_code_data',
f'stn_code_data_{yr}_{mm:02d}.mat'))['stn_mm']
if stn_yr is None: stn_yr = stn_mm
else: stn_yr = np.vstack([stn_yr, stn_mm])
matlab.savemat(
os.path.join(path_stn_jp, 'stn_code_data',
f'stn_code_data_{yr}_soramame.mat'), {'stn_yr': stn_yr})
print(yr)
