def do_mask_nan_utc(fname, key): read_name = os.path.join(path_goci_aod, fname) write_name = os.path.join(path_goci_filter, fname) arr = matlab.loadmat(read_name)[key] arr[nan_utc] = np.nan matlab.savemat(fname=write_name, data={key: arr})
def create_grid_goci_surrPx(grid_goci): DT = matlab.delaunayTriangulation(grid_goci) ti = DT.find_simplex(grid_kor) triPx = DT.simplices[ti, :] triPx_UL = np.min(triPx, axis=1) I, J = matlab.ind2sub(lon_goci.shape, triPx_UL) surrPx = np.zeros((len(I), 4)) surrPx[:, 0] = matlab.sub2ind(lon_goci.shape, I, J) # Upper Left surrPx[:, 1] = matlab.sub2ind(lon_goci.shape, I, J + 1) # Upper Right surrPx[:, 2] = matlab.sub2ind(lon_goci.shape, I + 1, J) # Lower Left surrPx[:, 3] = matlab.sub2ind(lon_goci.shape, I + 1, J + 1) # Lower Right lon_surr = lon_goci.T.flatten()[list( surrPx.flatten().astype(int))].reshape( surrPx.shape) # lon_goci[surrPx] lat_surr = lat_goci.T.flatten()[list( surrPx.flatten().astype(int))].reshape(surrPx.shape) diff_lon = lon_surr - np.tile(grid_kor[:, 0][:, None], (1, 4)) diff_lat = lat_surr - np.tile(grid_kor[:, 1][:, None], (1, 4)) d_surr = np.sqrt(np.power(diff_lon, 2) + np.power(diff_lat, 2)) invDsq = 1. / np.power(d_surr, 2) k = np.zeros((len(I), 1)) for i in range(len(I)): k[i] = surrPx[i, d_surr[i, :] == min(d_surr[i, :])] fname = os.path.join(path_processed_grid, 'grid_goci_surrPx.mat') matlab.savemat(fname, { 'surrPx': surrPx, 'd_surr': d_surr, 'invDsq': invDsq, 'k': k })
if weight_sum is None: weight_sum = weight else: weight_sum = np.vstack([weight_sum, weight]) min_dist = np.min(stn_GOCI6km_temp[:, 13]) stn_GOCI6km_temp2 = stn_GOCI6km_temp[ stn_GOCI6km_temp[:, 13] == min_dist, :] if stn_GOCI6km_temp2.shape[0] != 1: stn_GOCI6km_temp2 = stn_GOCI6km_temp2[ 0, :].reshape(1, -1) weight_sum = np.sum(weight_sum, axis=0) stn_GOCI6km_temp2[:, 5:11] = np.divide( np.nansum(stn_GOCI6km_temp[:, 5:11], axis=0), weight_sum) stn_GOCI6km = np.vstack( [stn_GOCI6km, stn_GOCI6km_temp2[:, :-1]]) stn_GOCI6km = stn_GOCI6km[ stn_GOCI6km[:, 12].argsort()] # sort by scode2 if stn_GOCI6km_yr is None: stn_GOCI6km_yr = stn_GOCI6km else: stn_GOCI6km_yr = np.vstack([stn_GOCI6km_yr, stn_GOCI6km]) print(doy) jp_stn_GOCI6km_yr = stn_GOCI6km_yr matlab.savemat(os.path.join(path_stn_jp, fname_save), {'jp_stn_GOCI6km_yr': jp_stn_GOCI6km_yr}) tElapsed = time.time() - tStart
else: mm = 12 for k in range(1,mm+1): ndata_temp2 = ndata_temp[ndata_temp[:,2]==k,:] if len(ndata_temp2)!=0: yrmon = yr*100+k idx = (stn_info_cn[j,4]<=yrmon)&(stn_info_cn[j,5]>=yrmon) if idx: # True ndata_temp2[:,-1]=stn_info_cn[j,1] if ndata_scode is None: ndata_scode = ndata_temp2 else: ndata_scode= np.vstack([ndata_scode,ndata_temp2]) if ((j+1)%100)==0: fname = f'stn_scode_data_{yr}_{j+1-99:04d}.mat' matlab.savemat(os.path.join(path_station,'Station_CN', fname), {'ndata_scode':ndata_scode}) ndata_scode = None elif (j+1)==stn_info_cn.shape[0]: if ndata_scode is not None: fname = f'stn_scode_data_{yr}_{np.int((j+1)/100)*100+1:04d}.mat' matlab.savemat(os.path.join(path_station,'Station_CN', fname), {'ndata_scode':ndata_scode}) print (f'{j+1} / {stn_info_cn.shape[0]}') ndata_scode = None file_list = glob.glob(os.path.join(path_station,'Station_CN', f'stn_scode_data_{yr}_*.mat')) file_list.sort() for fname in file_list: print (f'Reading ... {os.path.basename(fname)}') ndata_scode_temp = matlab.loadmat(fname)['ndata_scode'] print (ndata_scode_temp.shape) if ndata_scode is None:
YEARS = [2016] for yr in YEARS: flist = glob.glob( os.path.join(path_myd_processed, '03mask', str(yr), "*.tif")) flist.sort() for fname in flist: tStart = time.time() print(os.path.basename(fname)) with rio.open(fname) as src: ndvi = src.read(1) ndvi = np.float64(ndvi) ndvi[ndvi <= -32767] = np.nan ndvi = np.divide(ndvi, 10000) ndvi[np.abs(ndvi) >= 0.99] = np.nan values = ndvi.ravel(order='F') ndivi = griddata(points=points, values=values, xi=(lon_goci, lat_goci), method='linear') matlab.savemat( os.path.join( path_ea_goci_ndivi, str(yr), f'EA_MODIS_NDVI_{os.path.basename(fname)[10:-4]}.mat'), {'ndvi': ndivi}) del ndvi print(f'EA_MODIS_NDVI_{os.path.basename(fname)[10:-4]}.mat') tElapsed = time.time() - tStart print(f'time taken : {tElapsed}') print(yr)
'float64') GOCI_ndvi = matlab.hdfread( fname, 'NDVI_from_TOA_Reflectance_660_865nm').astype('float64') GOCI_dai = matlab.hdfread( fname, 'Dust_Aerosol_Index_from_412_443nm').astype('float64') GOCI_aod[(GOCI_aod < -0.05) | (GOCI_aod > 3.6)] = np.nan GOCI_fmf[(GOCI_fmf < 0) | (GOCI_fmf > 1)] = np.nan GOCI_ssa[(GOCI_ssa < 0) | (GOCI_ssa > 1)] = np.nan GOCI_ae[(GOCI_ae < 0) | (GOCI_ae > 3)] = np.nan GOCI_type[(GOCI_type < 0) | (GOCI_type > 6)] = np.nan GOCI_ndvi[(GOCI_ndvi < -1) | (GOCI_ndvi > 1)] = np.nan fname_temp = f'{yr}_{doy:03d}_{utc:02d}.mat' matlab.savemat(fname=os.path.join(path_goci_aod, 'AOD', str(yr), f'GOCI_AOD_{fname_temp}'), data={'GOCI_aod': GOCI_aod}) matlab.savemat(fname=os.path.join(path_goci_aod, 'FMF', str(yr), f'GOCI_FMF_{fname_temp}'), data={'GOCI_fmf': GOCI_fmf}) matlab.savemat(fname=os.path.join(path_goci_aod, 'SSA', str(yr), f'GOCI_SSA_{fname_temp}'), data={'GOCI_ssa': GOCI_ssa}) matlab.savemat(fname=os.path.join(path_goci_aod, 'AE', str(yr), f'GOCI_AE_{fname_temp}'), data={'GOCI_ae': GOCI_ae}) matlab.savemat(fname=os.path.join(path_goci_aod, 'Type', str(yr), f'GOCI_Type_{fname_temp}'), data={'GOCI_type': GOCI_type}) matlab.savemat(fname=os.path.join( path_goci_aod,
data = matlab.h5read( read_fname, '/HDFEOS/GRIDS/ColumnAmountNO2/Data Fields/ColumnAmountNO2TropCloudScreened' ) data = np.float64(data.T) # 720X1440 data[ data <= -1.2676506e+30] = np.nan # Assign NaN value to pixel that is out of valid range data = data * 3.7216e-17 data_yr[:, doy - 1] = data.ravel(order='F') out_fname = os.path.join(path_write, f'OMNO2d_trop_CS_{yr}_DU.mat') matlab.check_make_dir(os.path.dirname(out_fname)) data_yr[np.isnan(data_yr)] = -9999 matlab.savemat(out_fname, {'data_yr': data_yr}) tElapsed = time.time() - tStart print(f'{tElapsed} second') del data, data_yr print('==========================================================') ### OMSO2e print('OMSO2e') for yr in YEARS: tStart = time.time() doy_000 = matlab.datenum(f'{yr}0000') file_list = glob.glob( os.path.join(path_read, 'L3_grid', 'OMSO2e', str(yr), '*.he5')) file_list.sort() if yr % 4 == 0: days = 366
if i == days: data24 = matlab.loadmat( os.path.join(rdaps_path, str(yr + 1), f'RDAPS_{yr+1}_001_00.mat')) #rdaps else: data24 = matlab.loadmat( os.path.join(curr_path, f'RDAPS_{yr}_{i+1:03d}_00.mat')) # rdaps data24 = data24['rdaps'] for j in range(1, 5 + 1): rdaps = np.multiply((data06 - data00), (j / 6)) + data00 # 01 to 05 UTC fname = f'RDAPS_{yr}_{i:03d}_{j:02d}.mat' matlab.savemat(curr_path, fname, {'rdaps': rdaps}) rdaps = np.multiply((data12 - data06), (j / 6)) + data06 # 07 to 11 UTC fname = f'RDAPS_{yr}_{i:03d}_{j+6:02d}.mat' matlab.savemat(curr_path, fname, {'rdaps': rdaps}) rdaps = np.multiply((data18 - data12), (j / 6)) + data12 # 13 to 17 UTC fname = f'RDAPS_{yr}_{i:03d}_{j+12:02d}.mat' matlab.savemat(curr_path, fname, {'rdaps': rdaps}) rdaps = np.multiply((data24 - data18), (j / 6)) + data18 # 19 to 23 UTC fname = f'RDAPS_{yr}_{i:03d}_{j+18:02d}.mat' matlab.savemat(curr_path, fname, {'rdaps': rdaps})
doy000 = matlab.datenum(f'{yr}00000') date_list = dict() for x in range(1, days + 1): date_list[x] = matlab.datestr(doy000 + x) dates = df['doy'].apply(lambda x: date_list[x]) dates = pd.DatetimeIndex(dates) df['yr'] = dates.year df['mon'] = dates.month df['day'] = dates.day df.reset_index(drop=True, inplace=True) for col in cols: df.loc[df[col] >= 9997, col] = np.nan df.loc[df[col] == -9999, col] = np.nan df.dropna(axis=0, subset=cols, how='all', inplace=True) # axis=0: row, subset: 기준 ndata = df[header_ndata].values ind = np.lexsort( (ndata[:, 4], ndata[:, 0], ndata[:, 11])) # sort 11->0->4. 0 is primary ndata = ndata[ind] matlab.savemat( os.path.join(path_stn_jp, 'stn_code_data', f'stn_code_data_{yr}.mat'), {'ndata': ndata}) with h5py.File( os.path.join(path_stn_jp, 'stn_code_data', f'stn_code_data_{yr}.mat'), 'a') as dst: dst['header_ndata'] = header_ndata t2 = time.time() - t1 print(f'time taken : {t2:.3f}')
data2 = copy.deepcopy(temp.data) data2 = np.transpose(data2, (3, 2, 1, 0)) data2 = np.rot90(data2) emiss_all[:, :, j - 1, :2] = np.float64( np.squeeze(data2[:, :, 0, 1:3])) # forecast 01-02UTC except: pass for k in range(24): utc = k + 1 emiss = emiss_all[:, :, :, k] if k == 23: # last if doy == days: # last fname = f'emiss_27km_{yr+1}_001_00.mat' matlab.savemat( os.path.join(write_path, KNU_dir, str(yr + 1), fname), {'emiss': emiss}) else: fname = f'emiss_27km_{yr}_{doy+1:03d}_00.mat' matlab.savemat( os.path.join(write_path, KNU_dir, str(yr), fname), {'emiss': emiss}) else: fname = f'emiss_27km_{yr}_{doy:03d}_{utc:02d}.mat' matlab.savemat( os.path.join(write_path, KNU_dir, str(yr), fname), {'emiss': emiss}) print(f'{yr}_{doy:03d}') #except: # print (f'{yr}_{doy:03d}_no_data!!!!') # pass
# 고농도 지우기 if len(high) != 0: idx_high = np.where( data_stn[:, 25] <= high[-1, 25] & data_stn[:, 23] >= 150 & data_stn[:, -1] == 1)[0] data_stn = data_stn[~idx_high, :] elif high_rate > 30 and low_rate > 30: print(f'high_rate = {high_rate:3.2f} \n') print(f'low_rate = {low_rate:3.2f} \n') # 고농도 지우기 idx_high = np.where( data_stn[:, 25] <= high[-1, 25] & data_stn[:, 23] >= 150 & data_stn[:, -1] == 1)[0] ddata_stn = data_stn[~idx_high, :] # 저농도 지우기 idx_low = np.where( data_stn[:, 25] <= low[-1, 25] & data_stn[:, 23] <= 60 & data_stn[:, -1] == 1) data_stn = data_stn[~idx_low, :] # print (('data_stn 파일 갯수 줄이기.\n') # sample수 줄이는 코드 추가필요 전체적인 갯수줄이기 # data_stn[:,-1]==1 인 날에대해서 줄이기 print(utc) fname = f'{target[i]}_RTT_{yr}_{doy:03}_{utc:02d}.csv' tmp_df = pd.DataFrame(data_stn, columns=header2) matlab.savemat(os.path.join(path_rtt, 'dataset/', target[i], '/new/', fname), float_format='%7.7f') print(doy) print(yr)
lat_gpm, lon_gpm = mat['lat_gpm'], mat['lon_gpm'] del mat lat_gpm = lat_gpm[1100:1400, 2900:3300] # N50 W110 S20 E150 lon_gpm = lon_gpm[1100:1400, 2900:3300] points = np.array([lon_gpm.ravel(order='F'), lat_gpm.ravel(order='F')]).T del lon_gpm, lat_gpm print(f'points shape : {points.shape}') YEARS = [2016] for yr in YEARS: flist = glob.glob(os.path.join(path_gpm_processed, str(yr), '*.mat')) flist.sort() for fname in flist: tStart = time.time() precip = matlab.loadmat(fname)['precip'] precip = precip[1100:1400, 2900:3300] values = precip.ravel(order='F') precip = griddata(points=points, values=values, xi=(lon_goci, lat_goci), method='linear') matlab.savemat( os.path.join(path_ea_goci_gpm, str(yr), f'EA6km_{os.path.basename(fname)}'), {'precip': precip}) del precip print(os.path.basename(fname)) tElapsed = time.time() - tStart print(f'time taken : {tElapsed}') print(yr)
OX[nanidx[:, 2] == 1, :] = np.nan NO2[nanidx[:, 3] == 1, :] = np.nan PM10[nanidx[:, 4] == 1, :] = np.nan PM25[nanidx[:, 5] == 1, :] = np.nan allvar = np.vstack([ SO2.ravel(order='F'), CO.ravel(order='F'), OX.ravel(order='F'), NO2.ravel(order='F'), PM10.ravel(order='F'), PM25.ravel(order='F') ]).T nanidx_allvar = np.sum(np.isnan(allvar), axis=1) == 6 ndata_temp[:, 5:11] = allvar ndata_temp = ndata_temp[~(nanidx_allvar == 1)] if stn_JP is None: stn_JP = ndata_temp else: stn_JP = np.vstack([stn_JP, ndata_temp]) tElapsed_doy = time.time() - tStart_doy print(f'{yr}_{doy} -- {tElapsed_doy:3.4f} sec') else: print(f'Less than 4 hourly data in {doy:03d} (DOY) \n') matlab.savemat( os.path.join(path_stn_jp, 'stn_code_data', f'stn_code_data_rm_outlier_{yr}.mat'), {'stn_JP': stn_JP}) print(yr)
lon_goci, lat_goci = mat['lon_goci'], mat['lat_goci'] del mat coords = ((lon_goci - xmin) / dx, (lat_goci - ymin) / dy) # SRTM DEM with rio.open(os.path.join(path_grid_raw, 'SRTM_DEM_1km.tif')) as src: dem = src.read(1) dem = dem.astype('float64') dem[dem == src.nodata] = np.nan dem[dem < 0] = np.nan tStart = time.time() dem = map_coordinates(dem[::-1].T, coords, order=1) #dem = griddata(points, values, (lon_goci, lat_goci), method='linear') tElapsed = time.time() - tStart print(f'time taken : {tElapsed}') matlab.savemat(os.path.join(path_ea_goci, 'stationary', 'EA6km_SRTM_DEM.mat'), {'dem': dem}) print('stationary/EA6km_SRTM_DEM.mat') # Population density mat = matlab.loadmat( os.path.join(path_grid_raw, 'grid_EA_30sec_GPWv4_PopDens.mat')) lon_EA_30sec, lat_EA_30sec = mat['lon_EA_30sec'], mat['lat_EA_30sec'] del mat dx = 0.00833333 dy = 0.00833333 xmin = np.min(lon_EA_30sec[0]) ymin = np.min(lat_EA_30sec[:, 0]) coords = ((lon_goci - xmin) / dx, (lat_goci - ymin) / dy) popDens_all = np.zeros((3600, 4800, 21)) for yr in range(2000, 2020 + 1, 5):
dtype=h5py.string_dtype(encoding='utf-8')) ## stn_scode_data for South Korea YEARS = [2016] for yr in YEARS: if yr==2019: ndata=matlab.loadmat(os.path.join(path_stn_kor,'stn_code_data', 'stn_code_data_2019_010100_042300.mat'))['ndata'] else: ndata=matlab.loadmat(os.path.join(path_stn_kor,'stn_code_data', f'stn_code_data_{yr}.mat'))['ndata'] ndata = np.hstack([ndata, np.zeros([ndata.shape[0], 1])]) # add column for scode2 ndata[ndata<0] = np.nan ndata_scode = None # Assign scode2 for j in range(stn_info.shape[0]): ndata_temp = ndata[ndata[:,11]==stn_info[j,0],:] for k in range(1,12+1): ndata_temp2 = ndata_temp[ndata_temp[:,2]==k,:] if len(ndata_temp2)!=0: yrmon = yr*100+k idx = (stn_info[j,4]<=yrmon)&(stn_info[j,5]>yrmon) if idx: ndata_temp2[:,12]=stn_info[j,1] if ndata_scode is None: ndata_scode = ndata_temp2 else: ndata_scode=np.vstack([ndata_scode,ndata_temp2]) print (f'{j} / {stn_info.shape[0]}') fname = f'stn_scode_data_{yr}.mat' matlab.savemat(os.path.join(path_stn_kor, 'stn_scode_data', fname), {'ndata_scode':ndata_scode}) with h5py.File(os.path.join(path_stn_kor, 'stn_scode_data', fname), 'a') as dst: dst['header_ndata'] = header_ndata
stn[:, 0] = doy stn[:, 1] = yr stn[:, 2] = mm stn[:, 3] = dd stn[:, 4] = CST stn[:, 20] = scode_unq ttidx = np.sum(stn_hour == CST) #print (ttidx) if ttidx == 15: stn[:, 5:20] = stn_value[stn_hour == CST].T else: #nan, without time print(f'NO data in {CST:2.0f} (Local Time) on {doy:03d}') if stn_doy is None: stn_doy = stn else: stn_doy = np.concatenate((stn_doy, stn), axis=0) else: stn_doy = np.full((len(scode_unq) * 24, 21), np.nan) print(f'NO file in {doy:03d} (DOY)') if stn_yr is None: stn_yr = stn_doy else: stn_yr = np.concatenate((stn_yr, stn_doy), axis=0) fname = f'stn_code_data_{yr}.mat' matlab.savemat( os.path.join(path_station, 'Station_CN', 'stn_code_data', fname), {'stn_yr': stn_yr}) print(yr)
matlab.loadmat(os.path.join(path_stn_cn, 'Station_CN', fname)) # cn_stn_GOCI6km_yr cn_stn_GOCI6km_yr = mat['cn_stn_GOCI6km_yr'] del mat stn_GOCI6km_yr[:, 4] = stn_GOCI6km_yr[:, 4] - 9 cn_stn_GOCI6km_yr[:, 4] = cn_stn_GOCI6km_yr[:, 4] - 8 stn = np.concatenate( (stn_GOCI6km_yr, cn_stn_GOCI6km_yr[:, [0, 1, 2, 3, 4, 10, 18, 14, 12, 8, 6, 20, 21]]), axis=0) del stn_GOCI6km_location, dup_scode2_GOCI6km, header_stn_GOCI6km_location del cn_stn_GOCI6km_location, cn_dup_scode2_GOCI6km, header_cn_stn_GOCI6km_location del stn_GOCI6km_yr, cn_stn_GOCI6km_yr stn_nanidx = [] for doy in range(1, nanidx.shape[1] + 1): for utc in range(7 + 1): nanidx_temp = nanidx[:, doy - 1, utc] stn_temp = stn[stn[:, 0] == doy & stn[:, 4] == utc, :] for k in range(stn_temp.shape[0]): pid = stn_location[stn_location[:, 1] == stn_temp[k, 12], 4] stn_temp[k, 13] = nanidx_temp[pid] stn_nanidx = np.concatenate((stn_nanidx, stn_temp), axis=0) stn_fill = stn_nanidx[stn_nanidx[:, 13] == 0, :13] fname = f'stn_rf_day_{yr}.mat' matlab.savemat(os.path.join(path_ea_goci, fname), {'stn_fill': stn_fill}) print(yr)
if data_new is None: data_new = data_temp else: data_new = np.vstack([data_new, data_temp]) data_new[data_new[:, 7] >= 9997, 7] = np.nan if yr % 4 == 0: idx = (data_new[:, 2] == 2) & (data_new[:, 3] > 29) data_new = data_new[~idx] else: idx = (data_new[:, 2] == 2) & (data_new[:, 3] > 28) data_new = data_new[~idx] tmp_idx = (data_new[:, 2] == 4) & (data_new[:, 3] == 31) data_new = data_new[~tmp_idx] tmp_idx = (data_new[:, 2] == 6) & (data_new[:, 3] == 31) data_new = data_new[~tmp_idx] tmp_idx = (data_new[:, 2] == 9) & (data_new[:, 3] == 31) data_new = data_new[~tmp_idx] tmp_idx = (data_new[:, 2] == 11) & (data_new[:, 3] == 31) data_new = data_new[~tmp_idx] stn_tbl = pd.DataFrame(data_new, columns=header_p) stn_tbl = stn_tbl.apply(pd.to_numeric) matlab.savemat( os.path.join(path_stn_jp, 'byPollutant', f'JP_stn{varname}_{yr}.mat'), {col: stn_tbl[col].values for col in stn_tbl.columns}) with h5py.File( os.path.join(path_stn_jp, 'byPollutant', f'JP_stn{varname}_{yr}.mat'), 'a') as dst: dst['header'] = np.array(header_p, dtype=h5py.string_dtype(encoding='utf-8')) #stn_tbl.to_csv(os.path.join(path_stn_jp, f'JP_stn{varname}_{yr}.csv'),sep=',',na_rep='NaN')
mat = matlab.loadmat( os.path.join(path_korea_cases, f'nanidx_1km_hourly_{yr}')) # nanidx nanidx = mat['nanidx'] del mat # station mat = matlab.loadmat( os.path.join(path_stn_kr, f'Station_1km_rm_outlier_{yr}_weight.mat')) # stn_1km_yr stn_1km_yr = mat['stn_1km_yr'] del mat stn_1km_yr[:, 4] = stn_1km_yr[:, 4] - 9 # KST to UTC stn = stn_1km_yr stn_nanidx = [] for doy in range(1, nanidx.shape[1]): for utc in range(7 + 1): nanidx_temp = nanidx[:, doy - 1, utc] stn_temp = stn[stn[:, 0] == doy & stn[:, 4] == utc, :] for k in range(stn_temp.shape[0]): pid = stn_location[stn_location[:, 1] == stn_temp[k, 12], 4] stn_temp[k, 13] = nanidx_temp[pid] stn_nanidx = np.concatenate((stn_nanidx, stn_temp), axis=0) stn_fill = stn_nanidx[stn_nanidx[:, 13] == 0, :13] fname = f'KR_1km_stn_rf_day_{yr}.mat' matlab.savemat(os.path.join(path_korea_cases, fname), {'stn_fill': stn_fill}) print(yr)
stn_GOCI6km_temp[k, 5:11], weight) if weight_sum is None: weight_sum = weight else: weight_sum = np.vstack([weight_sum, weight]) min_dist = np.min(stn_GOCI6km_temp[:, 13]) stn_GOCI6km_temp2 = stn_GOCI6km_temp[ stn_GOCI6km_temp[:, 13] == min_dist, :] if stn_GOCI6km_temp2.shape[0] != 1: stn_GOCI6km_temp2 = stn_GOCI6km_temp2[0, :] # 픽셀중심에 더 가까운 관측소의 scode2를 사용하기 위함. 관측값은 가중평균한 값으로 다시 할당될거이므로 신경 쓰지말기 weight_sum = np.sum(weight_sum, axis=0) stn_GOCI6km_temp2[0, 5:11] = np.divide( np.nansum(stn_GOCI6km_temp[:, 5:11], axis=0), weight_sum) stn_GOCI6km = np.vstack( [stn_GOCI6km, stn_GOCI6km_temp2[:, :-1]]) stn_GOCI6km = stn_GOCI6km[ stn_GOCI6km[:, 12].argsort()] # sort by scode2 if stn_GOCI6km_yr is None: stn_GOCI6km_yr = stn_GOCI6km else: stn_GOCI6km_yr = np.vstack([stn_GOCI6km_yr, stn_GOCI6km]) print(doy) matlab.savemat( os.path.join(path_stn_kor, f'Station_GOCI6km_{yr}_weight.mat'), {'stn_GOCI6km_yr': stn_GOCI6km_yr}) tElapsed = time.time() - tStart print(f'time taken : {tElapsed}')
YEARS = [2016] for yr in YEARS: file_list = glob.glob(os.path.join(path_bess_raw, str(yr), '*.nc')) file_list.sort() for fname in file_list: tStart = time.time() print(f'Reading ... {os.path.basename(fname)}') if yr <= 2016: # Until 2016 (nc) ncfile = netCDF4.Dataset(fname) bess = np.array( ncfile.variables['surface_downwelling_shortwave_flux_in_air'] ).T ncfile.close() else: # 2017 (mat) bess = matlab.loadmat(fname)['bess'] bess = bess.astype('float64') bess[bess == -9999] = np.nan values = bess.ravel(order='F') tStart = time.time() RSDN = griddata(points, values, xi=(lon_goci, lat_goci), method='linear') write_fname = f'EA6km_BESS_RSDN_{yr}_{os.path.basename(fname)[-6:-3]}.mat' matlab.savemat(os.path.join(path_ea_goci_bess, str(yr), write_fname), {'RSDN': RSDN}) del bess, RSDN print(write_fname) tElapsed = time.time() - tStart print(f'time taken : {tElapsed}') print(yr)
# nan masking to prediction PM concentration. pred[nanidx==1,:] = np.nan if yr==2015 & doy==30: daily[0:218999,0:7] = np.nan daily[:,utc] = pred else: daily[:,utc] = pred print (utc) #close all except: print (f"{doy:03d}_{utc:02d}") # save hourly prediction matrix fname = f"{target[i]}_RTT_EA6km_{yr}_{doy:03d}" matlab.savemat(os.path.join(path_ea_goci, 'RF_map/',type_list[t],target[i],'daily/',fname), {'daily':daily}) #clear pred daily_avg = np.nanmean(daily,axis=1) if yr==2015: if len(annual)==0: annual = np.zeros((218999, 29))*np.nan annual[:,doy] = daily_avg else: annual[:,doy] = daily_avg # reshaping and mapping daily_avg = daily_avg.reshape(lon_goci.shape) fig, axs = matlab.m_kc(lon_goci,lat_goci,daily_avg) """
dup_scode2_GOCI6km = np.zeros((len(idx), np.max(unq_cnt) + 1)) dup_scode2_GOCI6km[:, 0] = idx for i in range(len(idx)): aa = stn[stn[:, 4] == dup_scode2_GOCI6km[i, 0], 1] dup_scode2_GOCI6km[i, 1:len(aa) + 1] = aa stn_GOCI6km_temp = stn[stn[:, 4] == idx[i], :] stn_GOCI6km_temp = np.hstack( [stn_GOCI6km_temp, np.ones([stn_GOCI6km_temp.shape[0], 1])]) stn_GOCI6km = np.vstack([stn_GOCI6km, stn_GOCI6km_temp]) stn_GOCI6km = stn_GOCI6km[stn_GOCI6km[:, 1].argsort()] jp_stn_GOCI6km_location = stn_GOCI6km jp_dup_scode2_GOCI6km = dup_scode2_GOCI6km header_jp_stn_GOCI6km_location = np.array( [ 'scode1', 'scode2', 'lat_org', 'lon_org', 'pxid', 'lat_px', 'lon_px', 'dist', 'avgid' ], dtype=h5py.string_dtype(encoding='utf-8')) matlab.savemat( os.path.join(path_stn_jp, 'jp_stn_GOCI6km_location_weight_v2017.mat'), { 'jp_stn_GOCI6km_location': jp_stn_GOCI6km_location, 'jp_dup_scode2_GOCI6km': jp_dup_scode2_GOCI6km }) with h5py.File( os.path.join(path_stn_jp, 'jp_stn_GOCI6km_location_weight_v2017.mat'), 'a') as dst: dst['header_jp_stn_GOCI6km_location'] = header_jp_stn_GOCI6km_location
typeOfLevel='heightAboveGround')[0].values rdaps[:, :, 11] = np.squeeze(data) data = rdaps_data.select(parameterName='Total precipitation')[0].values rdaps[:, :, 12] = np.squeeze(data) data = rdaps_data.select(name='Frictional velocity', typeOfLevel='heightAboveGround')[0].values rdaps[:, :, 13] = np.squeeze(data) data = rdaps_data.select( name='Convective available potential energy')[0].values rdaps[:, :, 14] = np.squeeze(data) data = rdaps_data.select(name='Surface roughness', typeOfLevel='surface')[0].values rdaps[:, :, 15] = np.squeeze(data) data = rdaps_data.select(name='Latent heat net flux', typeOfLevel='surface')[0].values rdaps[:, :, 16] = np.squeeze(data) data = rdaps_data.select(name='Specific humidity', typeOfLevel='heightAboveGround')[0].values rdaps[:, :, 17] = np.squeeze(data) doy = matlab.datenum(os.path.basename(fname)[21:29]) - doy_000 utc = int(os.path.basename(fname)[29:31]) #fname = f'RDAPS_{yr}_{doy:03d}_{utc:02d}_006.mat' fname = f'RDAPS_{yr}_{doy:03d}_{utc:02d}.mat' matlab.savemat(os.path.join(path_rdaps_processed, str(yr), fname), {'rdaps': rdaps}) print(fname) tElapsed = time.time() - tStart print(f'{tElapsed} second') print(yr)
dup_scode2_GOCI6km[:, 0] = idx for i in range(len(idx)): aa = stn[stn[:, 4] == dup_scode2_GOCI6km[i, 0], 1] dup_scode2_GOCI6km[i, 1:len(aa) + 1] = aa stn_GOCI6km_temp = stn[stn[:, 4] == idx[i], :] stn_GOCI6km_temp = np.hstack( [stn_GOCI6km_temp, np.ones([stn_GOCI6km_temp.shape[0], 1])]) stn_GOCI6km = np.vstack([stn_GOCI6km, stn_GOCI6km_temp]) stn_GOCI6km = stn_GOCI6km[stn_GOCI6km[:, 1].argsort()] cn_stn_GOCI6km_location = stn_GOCI6km cn_dup_scode2_GOCI6km = dup_scode2_GOCI6km header_cn_stn_GOCI6km_location = np.array( [ 'scode1', 'scode2', 'lat_org', 'lon_org', 'pxid', 'lat_px', 'lon_px', 'avgid', 'dist' ], dtype=h5py.string_dtype(encoding='utf-8')) matlab.savemat( os.path.join(path_station, 'Station_CN/cn_stn_GOCI6km_location_weight.mat'), { 'cn_stn_GOCI6km_location': cn_stn_GOCI6km_location, 'cn_dup_scode2_GOCI6km': cn_dup_scode2_GOCI6km }), with h5py.File( os.path.join(path_station, 'Station_CN', 'cn_stn_GOCI6km_location_weight.mat'), 'a') as dst: dst['header_cn_stn_GOCI6km_location'] = header_cn_stn_GOCI6km_location
if yr%4==0: days = 366 else: days = 365 for doy in range(1, days+1): # for doy=294:300 %1:days for utc in range(23+1): tStart = time.time() print (os.path.join(path_read, str(yr), f'emiss_27km_{yr}_{doy:03d}_{utc:02d}.mat')) try: emiss = matlab.loadmat(os.path.join(path_read, str(yr), f'emiss_27km_{yr}_{doy:03d}_{utc:02d}.mat'))['emiss'] EA_emis = np.zeros((nr_goci,nc_goci,14)) EA_emis[:,:,0]=griddata(points, emiss[:,:,0].ravel(order='F'),(lon_goci, lat_goci),method='linear') # ISOPRENE EA_emis[:,:,1]=griddata(points,emiss[:,:,1].ravel(order='F'),(lon_goci, lat_goci),method='linear') # TRP1 EA_emis[:,:,2]=griddata(points,emiss[:,:,4].ravel(order='F'),(lon_goci, lat_goci),method='linear') # CH4 EA_emis[:,:,3]=griddata(points,emiss[:,:,5].ravel(order='F'),(lon_goci, lat_goci),method='linear') # NO EA_emis[:,:,4]=griddata(points,emiss[:,:,6].ravel(order='F'),(lon_goci, lat_goci),method='linear') # NO2 EA_emis[:,:,6]=griddata(points,emiss[:,:,7].ravel(order='F'),(lon_goci, lat_goci),method='linear') # NH3 EA_emis[:,:,6]=griddata(points,emiss[:,:,9].ravel(order='F'),(lon_goci, lat_goci),method='linear') # HCOOH EA_emis[:,:,7]=griddata(points,emiss[:,:,10].ravel(order='F'),(lon_goci, lat_goci),method='linear') # HCHO EA_emis[:,:,8]=griddata(points,emiss[:,:,15].ravel(order='F'),(lon_goci, lat_goci),method='linear') # CO EA_emis[:,:,9]=griddata(points,emiss[:,:,36].ravel(order='F'),(lon_goci, lat_goci),method='linear') # SO2 EA_emis[:,:,10]=griddata(points,emiss[:,:,41].ravel(order='F'),(lon_goci, lat_goci),method='linear') # PMFINE EA_emis[:,:,11]=griddata(points,emiss[:,:,42].ravel(order='F'),(lon_goci, lat_goci),method='linear') # PNO3 EA_emis[:,:,12]=griddata(points,emiss[:,:,43].ravel(order='F'),(lon_goci, lat_goci),method='linear') # POA EA_emis[:,:,13]=griddata(points,emiss[:,:,44].ravel(order='F'),(lon_goci, lat_goci),method='linear') # PSO4 except IOError: print ('No data and make nan matrix file') EA_emis = np.full((nr_goci,nc_goci, 14), np.nan) pass matlab.savemat(os.path.join(path_ea_goci_emis, str(yr), f'EA6km_EMIS_{yr}_{doy:03d}_{utc:02d}.mat'), {'EA_emis':EA_emis}) print (f'EA6km_EMIS_{yr}_{doy:03d}_{utc:02d}') tElapsed = time.time() - tStart print (f'{tElapsed} second')
# points를 아래와 같이 배치 # (x=0,y=0), (x=0,y=1), ..., (x=0, y=n) # (x=1,y=0), (x=1,y=1), ... # (x=2,y=0), (x=2,y=1), ... class_name = [ 'barren', 'crop', 'forest', 'grass', 'savannas', 'shrub', 'snow', 'urban', 'water', 'wetland' ] YEARS = [2016] for yr in YEARS: for col in class_name: tStart = time.time() with rio.open( os.path.join(path_mcd_processed, '03_LC_ratio', str(yr), f'EA_{col}_ratio_r6_500m_{yr}.tif')) as src: LC_ratio = src.read(1) LC_ratio = np.float64(LC_ratio) LC_ratio[LC_ratio == src.nodata] = np.nan LC_ratio[LC_ratio < 0] = np.nan #% nodata value -3.40282346639e+038 LC_ratio = LC_ratio[::-1].T arr = map_coordinates(LC_ratio[::-1].T, coords, order=1) matlab.savemat( os.path.join(path_ea_goci_modis, f'EA6km_{col}_ratio_r6_{yr}.mat'), {f'LC_{col}': arr}) print(f'EA6km_{col}_ratio_r6_{yr}') del LC_ratio, arr tElapsed = time.time() - tStart print(f'time taken : {tElapsed}')
size = (1800, 3600, 48) gpm = np.zeros(size) doy = matlab.datenum(os.path.basename(list_temp[0])[21:29]) - doy0 + 1 print(f'doy: {doy}') for j, fname in enumerate(list_temp[:48]): gpm_temp = matlab.h5read(fname, '/Grid/precipitationCal') gpm_temp = np.float64(gpm_temp) gpm_temp[gpm_temp < -9999] = np.nan gpm[:, :, j] += gpm_temp precip = np.nansum(gpm, axis=2) precip *= 0.5 ap_fname = os.path.join(path_gpm_processed, str(yr), f'gpm_AP_{yr}_{doy:03d}_UTC00.mat') print(f'Saving ... {ap_fname}') matlab.savemat(ap_fname, {'precip': precip}) #for aa in range(2, len(list_gpm), 2): for aa in range(2, len(list_gpm) - 48 + 1, 2): tStart = time.time() gpm[:, :, 0:46] = gpm[:, :, 2:] gpm[:, :, -1] = 0 gpm[:, :, -2] = 0 #list_temp = list_gpm[aa:aa+2] list_temp = list_gpm[aa + 45:aa + 48] doy = matlab.datenum(os.path.basename(list_gpm[aa])[21:29]) - doy0 + 1 UTC = os.path.basename(list_gpm[aa])[31:33] for j in range(2): print(f'Reading ... {list_temp[j]}') gpm_temp = matlab.h5read(list_temp[j], '/Grid/precipitationCal')
@njit(error_model='numpy') def calculate(data): data_conv = np.full(data.shape, np.nan) for k in range(data.shape[0]): for t in range(data.shape[1]): t_ = t + 1 w_sum = 0 x_sum = 0 for n in range(data.shape[1]): n_ = n + 1 if np.isnan(data[k, n]) == 0: w = np.exp( (-(t_ - n_)**2) / (2 * sigma**2) ) # calcuate weights using t,n(days) with sigma # (k+1)-(n-1)=k-n x = data[k, n] * w w_sum += w x_sum += x data_conv[k, t] = x_sum / w_sum print(k) return data_conv data_conv = calculate(data) matlab.savemat( os.path.join( path_read, f'tempConv_{pname}_sigma{sigma}_{YEARS[0]}_{YEARS[len(YEARS)-1]}.mat' ), { 'data_conv': data_conv, 'data': data })
data_datenum = [matlab.datenum(val) for val in dstr] doy_000 = matlab.datenum(f'{yr}00000') doy = np.array([val - doy_000 for val in data_datenum]) stn_value = stn_tbl_temp[:, 2:15] data = np.hstack([ doy.reshape(-1, 1), dvec, stn_value[:, [0, 1, 5, 6, 3, 10, 11]], scode.reshape(-1, 1) ]) # {'doy','yr','mon','day','KST','SO2','CO','OX','NO2','PM10','PM25','scode'} if stn_mm is None: stn_mm = data else: stn_mm = np.vstack([stn_mm, data]) matlab.savemat( os.path.join(path_stn_jp, 'stn_code_data', f'stn_code_data_{yr}_{mm:02d}.mat'), {'stn_mm': stn_mm}) print(mm) print('Stack monthly data to yearly data') stn_yr = None for mm in range(1, 12 + 1): stn_mm = matlab.loadmat( os.path.join(path_stn_jp, 'stn_code_data', f'stn_code_data_{yr}_{mm:02d}.mat'))['stn_mm'] if stn_yr is None: stn_yr = stn_mm else: stn_yr = np.vstack([stn_yr, stn_mm]) matlab.savemat( os.path.join(path_stn_jp, 'stn_code_data', f'stn_code_data_{yr}_soramame.mat'), {'stn_yr': stn_yr}) print(yr)