def read_and_mask(yr):
if os.path.isfile(os.path.join(path_read, f'{pname}_{yr}_DU.mat')):
data_yr = matlab.loadmat(
os.path.join(path_read, f'{pname}_{yr}_DU.mat'))['data_yr']
elif os.path.isfile(os.path.join(path_read, f'{pname}_{yr}.mat')):
data_yr = matlab.loadmat(
os.path.join(path_read, f'{pname}_{yr}.mat'))['data_yr']
else:
raise ValueError(
f'{pname}_{yr}.mat does not exists though variation tried.')
data_subset = data_yr[mask == 1, :]
return data_subset
def do_mask_nan_utc(fname, key):
read_name = os.path.join(path_goci_aod, fname)
write_name = os.path.join(path_goci_filter, fname)
arr = matlab.loadmat(read_name)[key]
arr[nan_utc] = np.nan
matlab.savemat(fname=write_name, data={key: arr})
def read_table_mat(fname):
# should contain header column
mat = matlab.loadmat(fname)
tmp_cols = ['KST', 'scode', 'doy', list(mat.keys())[-2]]
df = pd.DataFrame(columns=tmp_cols)
for col in tmp_cols:
df[col] = np.squeeze(mat[col])
data = df.set_index(['KST', 'scode', 'doy'])
del df
print(os.path.basename(fname))
return data
import numpy as np
from scipy.interpolate import griddata
import time
### Setting path
data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_emis_processed = os.path.join(data_base_dir, 'Preprocessed_raw', 'EMIS')
path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km')
path_ea_goci_emis = os.path.join(path_ea_goci, 'EMIS')
KNU_dir = 'KNU_27_01'
path_read = os.path.join(path_emis_processed, KNU_dir)
#%% 27 km domain
mat = matlab.loadmat(os.path.join(path_grid_raw,'grid_cmaq_27km.mat')) # % lon_cmaq_27km, lat_cmaq_27km
points = np.array([mat['lon_cmaq_27km'].ravel(order='F'), mat['lat_cmaq_27km'].ravel(order='F')]).T
del mat
print (f'points shape : {points.shape}')
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat')) # lon_goci, lat_goci
lon_goci = mat['lon_goci']
lat_goci = mat['lat_goci']
del mat
nr_goci, nc_goci = lon_goci.shape
YEARS = [2016]
for yr in YEARS: #:2016
if yr%4==0: days = 366
else: days = 365
for doy in range(1, days+1): # for doy=294:300 %1:days
if yr == 2019:
days = 148
list_aod = glob.glob(os.path.join(path_goci_aod, 'AOD', str(yr), '*.mat'))
list_aod = [os.path.basename(f) for f in list_aod]
list_aod.sort()
list_date = [x[9:20] for x in list_aod]
for doy in range(1, days + 1):
nan_filter = np.zeros((473, 463, 8, 4))
data = np.full([473, 463, 8], np.nan)
for utc in range(num_utc):
tStart = time.time()
fname = os.path.join(path_goci_aod, 'AOD', str(yr),
f'GOCI_AOD_{yr}_{doy:03d}_{utc:02d}.mat')
GOCI_aod = matlab.loadmat(fname)['GOCI_aod']
data[:, :, utc] = GOCI_aod
fname = os.path.join(
path_goci_aod,
'No_of_Used_500m_Pixels_for_One_6km_Product_Pixel', str(yr),
f'GOCI_num_used_pixels_{yr}_{doy:03d}_{utc:02d}.mat')
GOCI_num_used_pixels = matlab.loadmat(
fname)['GOCI_num_used_pixels']
GOCI_aod_bfr = np.full([477, 467],
np.nan) # buffer for 5x5 moving windows
GOCI_aod_bfr[2:-2, 2:-2] = GOCI_aod
nan_filter = moving_window_2d(utc, GOCI_aod_bfr, nan_filter)
data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km')
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
path_stn_jp = os.path.join(path_station, 'Station_JP')
path_stn_cn = os.path.join(path_station, 'Station_CN')
path_stn_kr = os.path.join(path_station, 'Station_KR')
path_output = os.path.join(data_base_dir, 'output', 'RealTimeTraining', 'EastAsia')
matlab.check_make_dir(path_output)
tg = ['PM10','PM25']
## Load grid
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat'))
lon_goci, lat_goci = mat['lon_goci'], mat['lat_goci']
del mat
##
YEARS = [2016]
for yr in YEARS:
#mat = matlab.loadmat(os.path.join(path_stn_kr, 'stn_GOCI6km_location_weight_v2018.mat'))
mat = matlab.loadmat(os.path.join(path_stn_kr, 'stn_GOCI6km_location_weight_v201904.mat'))
dup_scode2_GOCI6km = mat['dup_scode2_GOCI6km']
df = pd.DataFrame(mat['stn_GOCI6km_location'], columns=mat['header_stn_GOCI6km_location'])
stn_GOCI6km_location = df.values
del df, mat
mat = matlab.loadmat(os.path.join(path_stn_cn, 'cn_stn_GOCI6km_location_weight.mat'))
cn_dup_scode2_GOCI6km = mat['cn_dup_scode2_GOCI6km']
from Code.utils import matlab
import numpy as np
import rasterio as rio
from scipy.ndimage import map_coordinates
import time
### Setting path
data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_mcd_processed = os.path.join(data_base_dir, 'Preprocessed_raw', 'MODIS',
'MCD12Q1')
path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km')
path_ea_goci_modis = os.path.join(path_ea_goci, 'MODIS_LC_ratio')
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_GCS_MODIS_500m_EA.mat'))
lon_gcs_500m, lat_gcs_500m = mat['lon_gcs_500m'], mat['lat_gcs_500m']
del mat
dx = 0.00511542 # point x 간격 np.diff(mat['lon_gcs_500m'])
dy = 0.00511542 # point y 간격 np.diff(mat['lat_gcs_500m'])
xmin = np.min(lon_gcs_500m[0])
ymin = np.min(lat_gcs_500m[:, 0])
mat = matlab.loadmat(os.path.join(path_grid_raw,
'grid_goci.mat')) # lon_goci, lat_goci
lon_goci = mat['lon_goci']
lat_goci = mat['lat_goci']
del mat
coords = ((lon_goci - xmin) / dx, (lat_goci - ymin) / dy)
# points를 아래와 같이 배치
import pandas as pd
import glob
import h5py
### Setting path
#data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
#path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
#data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
data_base_dir = os.path.join('/', 'share', 'irisnas5', 'GEMS', 'GEMS_python')
#data_base_dir = os.path.join('//','10.72.26.56','irisnas5', 'GEMS', 'GEMS_python')
#path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_grid_raw = os.path.join('/', 'share', 'irisnas5', 'Data', 'grid')
#path_grid_raw = os.path.join('//','10.72.26.56','irisnas5','Data','grid')
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat'))
latlon_data = np.array(
[mat['lat_goci'].ravel(order='F'), mat['lon_goci'].ravel(order='F')]).T
del mat
## China
# scode1, scode2, lon, lat, op_start, op_
stn_info_cn = pd.read_csv(
os.path.join(path_station, 'Station_CN',
'cn_stn_code_lonlat_period_GOCI.csv'))
stn_info_cn = stn_info_cn.values
new_station = np.zeros((stn_info_cn.shape[0], 4))
for i in range(stn_info_cn.shape[0]):
dist = copy.deepcopy(latlon_data)
dist[:, 0] = dist[:, 0] - stn_info_cn[i, 3]
import netCDF4
import numpy as np
import glob
from scipy.interpolate import griddata
import time
### Setting path
data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_bess_raw = os.path.join(data_base_dir, 'Raw', 'BESS') # Read
path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km')
path_ea_goci_bess = os.path.join(path_ea_goci, 'Preprocessed_raw',
'EA_GOCI6km', 'BESS') # Write
mat = matlab.loadmat(os.path.join(path_grid_raw,
'grid_bess.mat')) # lon_bess, lat_bess
points = np.array(
[mat['lon_bess'].ravel(order='F'), mat['lat_bess'].ravel(order='F')]).T
del mat
print(f'points shape : {points.shape}')
mat = matlab.loadmat(os.path.join(path_grid_raw,
'grid_goci.mat')) # lon_goci, lat_goci
lon_goci = mat['lon_goci']
lat_goci = mat['lat_goci']
del mat
YEARS = [2016]
for yr in YEARS:
file_list = glob.glob(os.path.join(path_bess_raw, str(yr), '*.nc'))
file_list.sort()
#data_path = os.path.join(project_data, 'Data', 'Aerosol')
data_base_dir = os.path.join('/', 'share', 'irisnas5', 'GEMS',
'GEMS_python') # revised
rdaps_path = os.path.join(data_base_dir, 'Preprocessed_raw',
'RDAPS') # revised
### Setting period
YEARS = [2016] #, 2018, 2019
for yr in YEARS:
curr_path = os.path.join(path_rdaps_processed, str(yr))
if yr % 4 == 0: days = 366
else: days = 365
for i in range(1, days + 1):
data00 = matlab.loadmat(
os.path.join(curr_path, f'RDAPS_{yr}_{i:03d}_00.mat')) # rdaps
data00 = data00['rdaps']
data06 = matlab.loadmat(
os.path.join(curr_path, f'RDAPS_{yr}_{i:03d}_06.mat')) # rdaps
data06 = data06['rdaps']
data12 = matlab.loadmat(
os.path.join(curr_path, f'RDAPS_{yr}_{i:03d}_12.mat')) # rdaps
data12 = data12['rdaps']
data18 = matlab.loadmat(
os.path.join(curr_path, f'RDAPS_{yr}_{i:03d}_18.mat')) # rdaps
data18 = data18['rdaps']
if i == days:
### Setting path
data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km')
path_nox_korea = os.path.join(data_base_dir, 'Preprocessed_raw', 'NOX03', 'Korea')
matlab.check_make_dir(path_nox_korea)
path_korea_cases = os.path.join(data_base_dir, 'Preprocessed_raw', 'Korea', 'cases')
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
path_stn_jp = os.path.join(path_station, 'Station_JP')
path_stn_cn = os.path.join(path_station, 'Station_CN')
path_stn_kr = os.path.join(path_station, 'Station_KR')
## Load grid
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_korea.mat'))
lat_kor, lon_kor = mat['lat_kor'], mat['lon_kor']
del mat
path_data = '/share/irisnas6/Data/Aerosol/'
path = '/share/irisnas6/Work/NOXO3/Korea/'
#m_bnd = m_shaperead([path_nox_korea,'boundary/BND_SIDO_GCS'])
shape_feature = ShapelyFeature(Reader(os.path.join(path_nox_korea, 'boundary', 'BND_SIDO_GCS.shp')).geometries(),
ccrs.PlateCarree(), edgecolor='black')
# sel_NO2 = [1,3:4,7:17,21:24,28:49,53]
# sel_O3 = [1:23,25,28:49,53]
#yr=2015
yr = 2016
import glob
import numpy as np
from scipy.interpolate import griddata
import time
### Setting path
data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_gpm_processed = os.path.join(data_base_dir, 'Preprocessed_raw', 'GPM',
'AP_24h_hourly') # read
path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km')
path_ea_goci_gpm = os.path.join(path_ea_goci, 'Preprocessed_raw', 'EA_GOCI6km',
'GPM_AP') # Write
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat'))
lat_goci, lon_goci = mat['lat_goci'], mat['lon_goci']
del mat
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_gpm.mat'))
lat_gpm, lon_gpm = mat['lat_gpm'], mat['lon_gpm']
del mat
lat_gpm = lat_gpm[1100:1400, 2900:3300] # N50 W110 S20 E150
lon_gpm = lon_gpm[1100:1400, 2900:3300]
points = np.array([lon_gpm.ravel(order='F'), lat_gpm.ravel(order='F')]).T
del lon_gpm, lat_gpm
print(f'points shape : {points.shape}')
YEARS = [2016]
for yr in YEARS:
flist = glob.glob(os.path.join(path_gpm_processed, str(yr), '*.mat'))
## header
# station_header
header = [
'doy', 'yr', 'mon', 'day', 'KST', 'SO2', 'CO', 'OX', 'NO2', 'PM10', 'PM25',
'scode'
]
##
YEARS = [2016]
for yr in YEARS:
if yr % 4 == 0: days = 366
else: days = 365
#if yr==2019: days=151;
ndata = matlab.loadmat(
os.path.join(path_stn_jp, 'stn_code_data',
f'stn_code_data_{yr}_soramame.mat'))['stn_yr']
ndata = ndata.astype('float')
scode = np.unique(ndata[:, -1])
print(ndata.shape)
# SO2
ndata[:, 5] = ndata[:, 5] * 1000
# ppm to ppb
ndata[ndata[:, 5] > 400, 5] = np.nan
# CO
ndata[:, 6] = ndata[:, 6] * 10
# ppm to 0.1ppm
ndata[ndata[:, 6] > 200, 6] = np.nan
# OX
ndata[:, 7] = ndata[:, 7] * 1000
# ppm to ppb
base_dir = os.environ['GEMS_HOME']
project_path = os.path.join(base_dir, 'python-refactor')
sys.path.insert(0, project_path)
from Code.utils import matlab
import time
import numpy as np
import rasterio as rio
from scipy.ndimage import map_coordinates
### Setting path
data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km')
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_SRTM_1km.mat'))
lon_SRTM_1km, lat_SRTM_1km = mat['lon_SRTM_1km'], mat['lat_SRTM_1km']
del mat
dx = 0.01023084 # point x 간격 np.diff(mat['lon_gcs_500m'])
dy = 0.01023084 # point y 간격 np.diff(mat['lat_gcs_500m'])
xmin = np.min(lon_SRTM_1km[0])
ymin = np.min(lat_SRTM_1km[:, 0])
mat = matlab.loadmat(os.path.join(path_grid_raw,
'grid_goci.mat')) # lon_goci, lat_goci
lon_goci, lat_goci = mat['lon_goci'], mat['lat_goci']
del mat
coords = ((lon_goci - xmin) / dx, (lat_goci - ymin) / dy)
# SRTM DEM
with rio.open(os.path.join(path_grid_raw, 'SRTM_DEM_1km.tif')) as src:
path_in_situ = os.path.join(data_base_dir, 'Raw')
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
path_stn_kor = os.path.join(path_station, 'Station_KR')
stn_info = pd.read_csv(os.path.join(path_stn_kor, 'stn_code_lonlat_period_2005_201904.csv'))
stn_info = stn_info.values
# scode1, scode2, lon, lat, op_start, op_
## read raw files
header_ndata = np.array(['doy','yr','mon','day','KST','SO2','CO','O3','NO2','PM10','PM25','scode','scode2'],
dtype=h5py.string_dtype(encoding='utf-8'))
## stn_scode_data for South Korea
YEARS = [2016]
for yr in YEARS:
if yr==2019:
ndata=matlab.loadmat(os.path.join(path_stn_kor,'stn_code_data', 'stn_code_data_2019_010100_042300.mat'))['ndata']
else:
ndata=matlab.loadmat(os.path.join(path_stn_kor,'stn_code_data', f'stn_code_data_{yr}.mat'))['ndata']
ndata = np.hstack([ndata, np.zeros([ndata.shape[0], 1])]) # add column for scode2
ndata[ndata<0] = np.nan
ndata_scode = None
# Assign scode2
for j in range(stn_info.shape[0]):
ndata_temp = ndata[ndata[:,11]==stn_info[j,0],:]
for k in range(1,12+1):
ndata_temp2 = ndata_temp[ndata_temp[:,2]==k,:]
if len(ndata_temp2)!=0:
yrmon = yr*100+k
idx = (stn_info[j,4]<=yrmon)&(stn_info[j,5]>yrmon)
if idx:
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km')
path_korea_cases = os.path.join(data_base_dir, 'Preprocessed_raw', 'Korea',
'cases')
matlab.check_make_dir(path_korea_cases)
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
path_stn_jp = os.path.join(path_station, 'Station_JP')
path_stn_cn = os.path.join(path_station, 'Station_CN')
path_stn_kr = os.path.join(path_station, 'Station_KR')
tg = ['PM10', 'PM25']
## Load grid
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_korea.mat'))
lat_kor, lon_kor = mat['lat_kor'], mat['lon_kor']
del mat
mat = matlab.loadmat(
os.path.join(path_stn_kr,
'stn_1km_location_weight_v201904.mat')) # stn_1km_location
stn_1km_location = mat['stn_1km_location']
del mat
# scode1,scode2, lat_org, lon_org, pxid, lat_px, lon_px
stn_location = stn_1km_location
# scode1,scode2, lat_org, lon_org, pxid, lat_px, lon_px
##
YEARS = [2016]
sys.path.insert(0, project_path)
from Code.utils import matlab
import time
import numpy as np
from scipy.interpolate import griddata
### Setting path
data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_omi_processed = os.path.join(data_base_dir, 'Preprocessed_raw',
'OMI') # Read
path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km')
path_ea_goci_omi = os.path.join(path_ea_goci, 'OMI_tempConv') # Write
mat = matlab.loadmat(os.path.join(path_grid_raw,
'grid_goci.mat')) # lon_goci, lat_goci
lon_goci, lat_goci = mat['lon_goci'], mat['lat_goci']
del mat
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_omi_25.mat'))
lon_omi, lat_omi = mat['lon_omi'], mat['lat_omi']
del mat
lon_omi = lon_omi[340:560, 1020:1320]
lat_omi = lat_omi[340:560, 1020, 1320]
points = np.array([lon_omi.ravel(order='F'), lat_omi.ravel(order='F')]).T
del lon_omi, lat_omi
print(f'points shape : {points.shape}')
dlist = list(
doy_000 = matlab.datenum(f'{yr}00000')
doy = np.array([val - doy_000 for val in data_datenum])
stn_value = stn_tbl_temp[:, 2:15]
data = np.hstack([
doy.reshape(-1, 1), dvec, stn_value[:,
[0, 1, 5, 6, 3, 10, 11]],
scode.reshape(-1, 1)
])
# {'doy','yr','mon','day','KST','SO2','CO','OX','NO2','PM10','PM25','scode'}
if stn_mm is None: stn_mm = data
else: stn_mm = np.vstack([stn_mm, data])
matlab.savemat(
os.path.join(path_stn_jp, 'stn_code_data',
f'stn_code_data_{yr}_{mm:02d}.mat'),
{'stn_mm': stn_mm})
print(mm)
print('Stack monthly data to yearly data')
stn_yr = None
for mm in range(1, 12 + 1):
stn_mm = matlab.loadmat(
os.path.join(path_stn_jp, 'stn_code_data',
f'stn_code_data_{yr}_{mm:02d}.mat'))['stn_mm']
if stn_yr is None: stn_yr = stn_mm
else: stn_yr = np.vstack([stn_yr, stn_mm])
matlab.savemat(
os.path.join(path_stn_jp, 'stn_code_data',
f'stn_code_data_{yr}_soramame.mat'), {'stn_yr': stn_yr})
print(yr)
import pandas as pd
### Setting path
data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km')
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
path_stn_jp = os.path.join(path_station, 'Station_JP')
path_stn_cn = os.path.join(path_station, 'Station_CN')
path_stn_kr = os.path.join(path_station, 'Station_KR')
tg = ['PM10', 'PM25']
## Load grid
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat'))
lon_goci, lat_goci = mat['lon_goci'], mat['lat_goci']
del mat
YEARS = [2016]
for yr in YEARS:
#mat = matlab.loadmat(os.path.join(path_stn_kr, 'stn_GOCI6km_location_weight_v2018.mat'))
mat = matlab.loadmat(
os.path.join(path_stn_kr, 'stn_GOCI6km_location_weight_v201904.mat'))
dup_scode2_GOCI6km = mat['dup_scode2_GOCI6km']
df = pd.DataFrame(mat['stn_GOCI6km_location'],
columns=mat['header_stn_GOCI6km_location'])
stn_GOCI6km_location = df.values
del df, mat
mat = matlab.loadmat(
'cases')
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
path_stn_jp = os.path.join(path_station, 'Station_JP')
path_stn_cn = os.path.join(path_station, 'Station_CN')
path_stn_kr = os.path.join(path_station, 'Station_KR')
path_data = '/share/irisnas5/Data/'
path = '/share/irisnas5/GEMS/PM/00_EA6km/'
season_name = ['Spring', 'Summer', 'Fall', 'Winter']
season_name2 = ['MAM', 'JJA', 'SON', 'DJF']
type_list = ['conc', 'time', 'time_conc']
target = ['PM10', 'PM25']
## Load grid
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_korea.mat'))
lat_kor, lon_kor = mat['lat_kor'], mat['lon_kor']
del mat
season_num = [61, 153, 245, 336]
YEARS = [2016]
for t in [0, 2]: #1:2:3
for i in [0, 1]:
for yr in YEARS: #2015:2017
if yr % 4 == 0:
days = 366
daysInMonths = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
else:
days = 365
daysInMonths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
fname = f"{target[i]}_RTT_EA6km_{yr}.mat"
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km')
path_korea_cases = os.path.join(data_base_dir, 'Preprocessed_raw', 'Korea',
'cases')
matlab.check_make_dir(path_korea_cases)
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
path_stn_jp = os.path.join(path_station, 'Station_JP')
path_stn_cn = os.path.join(path_station, 'Station_CN')
path_stn_kr = os.path.join(path_station, 'Station_KR')
tg = ['PM10', 'PM25']
## Load grid
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_korea.mat'))
lat_kor, lon_kor = mat['lat_kor'], mat['lon_kor']
del mat
mat = matlab.loadmat(
os.path.join(path_stn_kr,
'stn_1km_location_weight_v201904.mat')) # stn_1km_location
stn_1km_location = mat['stn_1km_location']
del mat
# scode1,scode2, lat_org, lon_org, pxid, lat_px, lon_px
##
YEARS = [2016]
for yr in YEARS:
# nanidx for cases
mat = matlab.loadmat(
sys.path.insert(0, project_path)
from Code.utils import matlab
import copy
import numpy as np
import pandas as pd
import glob
import time
### Setting path
data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
path_stn_kor = os.path.join(path_station, 'Station_KR')
## South Korea
mat = matlab.loadmat(
os.path.join(path_stn_kor, 'stn_GOCI6km_location_weight_v201904.mat'))
dup_scode2_GOCI6km, stn_GOCI6km_location = mat['dup_scode2_GOCI6km'], mat[
'stn_GOCI6km_location']
del mat
dup_scode2 = dup_scode2_GOCI6km[:, 1:]
unq_scode2 = stn_GOCI6km_location[stn_GOCI6km_location[:, 8] == 0, 1]
idx = [val in dup_scode2 for val in stn_GOCI6km_location[:, 1]]
dup_dist = stn_GOCI6km_location[idx][:, [1, 7]]
YEARS = [2016] #2005:2019
for yr in YEARS:
tStart = time.time()
ndata_scode = matlab.loadmat(
os.path.join(path_stn_kor, 'stn_scode_data',
f'stn_scode_data_{yr}.mat'))['ndata_scode']
import random
### Setting path
data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km')
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
path_stn_jp = os.path.join(path_station, 'Station_JP')
path_stn_cn = os.path.join(path_station, 'Station_CN')
path_stn_kr = os.path.join(path_station, 'Station_KR')
path_rtt = os.path.join(data_base_dir, 'Preprocessed_raw', 'RTT') # path_save
## Station index
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_korea.mat'))
lat_kor, lon_kor = mat['lat_kor'], mat['lon_kor']
del mat
mat = matlab.loadmat(os.path.join(path_stn_kr, 'stn_1km_location_weight_v201904.mat'))
dup_scode2_1km = mat['dup_scode2_1km']
df = pd.DataFrame(mat['stn_1km_location'], columns=mat['header_stn_1km_location'][0])
stn_1km_location = df.values
del df, mat
## Read data
target = ['PM10','PM25']
# cd([path_nas4,'cases/RTT/'])
i=2 #1:2 # target ########################################
header = ['NDVI','AOD','AE','FMF','SSA','RSDN','Precip','DEM','LC_ratio', # satellite data(9)
path_korea_cases = os.path.join(data_base_dir, 'Preprocessed_raw', 'Korea', 'cases')
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
path_stn_jp = os.path.join(path_station, 'Station_JP')
path_stn_cn = os.path.join(path_station, 'Station_CN')
path_stn_kr = os.path.join(path_station, 'Station_KR')
path_data = '/share/irisnas5/Data/'
path= '/share/irisnas5/GEMS/PM/00_EA6km/'
target = ['PM10','PM25']
type_list = ['conc','time','time_conc']
YEARS = [2016]
## Load grid
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_korea.mat'))
lat_kor, lon_kor = mat['lat_kor'], mat['lon_kor']
del mat
for t in [1]:
for i in [0]:
for yr in YEARS:
if yr%4==0:
days = 366
daysInMonths = [31,29,31,30,31,30,31,31,30,31,30,31]
else:
days = 365
daysInMonths = [31,28,31,30,31,30,31,31,30,31,30,31]
daily = []
monthly=[]
annual =[]
data_base_dir = os.path.join('/', 'share', 'irisnas5', 'GEMS', 'GEMS_python')
#data_base_dir = os.path.join('//', '10.72.26.56','irisnas5', 'GEMS', 'GEMS_python')
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
# scode1, scode2, lon, lat, op_start, op_
stn_info_cn = pd.read_csv(os.path.join(path_station, 'Station_CN', 'cn_stn_code_lonlat_period.csv'))
stn_info_cn = stn_info_cn.values
## stn_scode_data for China
header_ndata = np.array(['doy','yr','mon','day','CST','AQI','PM25','PM25_24h',
'PM10','PM10_24h','SO2','SO2_24h','NO2','NO2_24h','O3','O3_24h',
'O3_8h','O3_8h_24h','CO','CO_24h','scode','scode2'], dtype=h5py.string_dtype(encoding='utf-8'))
YEARS = [2016] # range(2015, 2019+1)
for yr in YEARS:
fname = f'stn_code_data_rm_outlier_{yr}.mat'
ndata = matlab.loadmat(os.path.join(path_station,'Station_CN', 'stn_code_data', fname))['stn_CN']
ndata = np.hstack([ndata, np.zeros([ndata.shape[0], 1])])
ndata_scode = None
# Assign scode2
for j in range(stn_info_cn.shape[0]):
ndata_temp = ndata[ndata[:,-2]==stn_info_cn[j,0],:]
if yr==2019:
mm = 5
else:
mm = 12
for k in range(1,mm+1):
ndata_temp2 = ndata_temp[ndata_temp[:,2]==k,:]
if len(ndata_temp2)!=0:
yrmon = yr*100+k
sys.path.insert(0, project_path)
from Code.utils import matlab
import glob
import numpy as np
from scipy.interpolate import griddata
import time
### Setting path
data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_rdaps_processed = os.path.join(data_base_dir, 'Preprocessed_raw', 'RDAPS')
path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km')
path_ea_goci_rdaps = os.path.join(path_ea_goci, 'RDAPS')
mat = matlab.loadmat(os.path.join(path_grid_raw,
'grid_goci.mat')) # lon_goci, lat_goci
lon_goci, lat_goci = mat['lon_goci'], mat['lat_goci']
del mat
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_rdaps.mat'))
points = np.array(
[mat['lon_rdaps'].ravel(order='F'), mat['lat_rdaps'].ravel(order='F')]).T
del mat
print(f'points shape : {points.shape}')
YEARS = [2016]
for yr in YEARS:
flist = glob.glob(os.path.join(path_rdaps_processed, str(yr), '*.mat'))
flist.sort()
for fname in flist: # in range(1, 2000+1):
tStart = time.time()
import copy
import numpy as np
import pandas as pd
import glob
import time
### Setting path
#data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
#data_base_dir = os.path.join('//', '10.72.26.56','irisnas5', 'GEMS', 'GEMS_python')
data_base_dir = os.path.join('/', 'share', 'irisnas5', 'GEMS', 'GEMS_python')
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
path_stn_jp = os.path.join(path_station, 'Station_JP')
## Japan
mat = matlab.loadmat(
os.path.join(path_stn_jp, 'jp_stn_GOCI6km_location_weight_v2017.mat'))
jp_dup_scode2_GOCI6km, jp_stn_GOCI6km_location = mat[
'jp_dup_scode2_GOCI6km'], mat['jp_stn_GOCI6km_location']
del mat
dup_scode2 = jp_dup_scode2_GOCI6km[:, 1:]
unq_scode2 = jp_stn_GOCI6km_location[jp_stn_GOCI6km_location[:, 8] == 0, 1]
idx = [val in dup_scode2 for val in jp_stn_GOCI6km_location[:, 1]]
dup_dist = jp_stn_GOCI6km_location[idx][:, [1, 7]]
YEARS = [2016]
for yr in YEARS:
tStart = time.time()
if os.path.isfile(
os.path.join(path_stn_jp, 'stn_scode_data',
f'jp_stn_scode_data_{yr}.mat')):
import random
### Setting path
data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid')
path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km')
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
path_stn_jp = os.path.join(path_station, 'Station_JP')
path_stn_cn = os.path.join(path_station, 'Station_CN')
path_stn_kr = os.path.join(path_station, 'Station_KR')
path_rtt = os.path.join(data_base_dir, 'Preprocessed_raw', 'RTT') # path_save
## Station index
mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat'))
lon_goci, lat_goci = mat['lon_goci'], mat['lat_goci']
del mat
mat = matlab.loadmat(
os.path.join(path_stn_cn, 'cn_stn_GOCI6km_location_weight.mat'))
cn_dup_scode2_GOCI6km = mat['cn_dup_scode2_GOCI6km']
header_cn_stn_GOCI6km_location = mat['header_cn_stn_GOCI6km_location']
df = pd.DataFrame(mat['cn_stn_GOCI6km_location'],
columns=header_cn_stn_GOCI6km_location)
cn_stn_GOCI6km_location = df.values
del df, mat
#mat = matlab.loadmat(os.path.join(path_stn_kr, 'stn_GOCI6km_location_weight_v2018.mat'))
mat = matlab.loadmat(
os.path.join(path_stn_kr, 'stn_GOCI6km_location_weight_v201904.mat'))
#data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
#path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
#data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
#path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
data_base_dir = os.path.join('/', 'share', 'irisnas5', 'GEMS', 'GEMS_python')
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
YEARS = [2016] # range(2015, 2019+1)
for yr in YEARS:
if yr % 4 == 0: days = 366
else: days = 365
if yr == 2019: days = 151
ndata = matlab.loadmat(
os.path.join(path_station, 'Station_CN', 'stn_code_data',
f'stn_code_data_{yr}.mat'))['stn_yr']
scode = np.unique(ndata[:, -1])
# CO
ndata[:, 18] = ndata[:, 18] / 1.15 # (mg/m3) to ppm (1 ppm = 1.15 mg m-3)
ndata[ndata[:, 18] > 20, 18] = np.nan
# SO2
ndata[:, 10] = ndata[:, 10] / 2.62
# (ug/m3) to ppb (1 ppb = 2.62 ug m-3)
ndata[ndata[:, 10] > 400, 10] = np.nan
# NO2
ndata[:, 12] = ndata[:, 12] / 1.88
# (ug/m3) to ppb (1 ppb = 1.88 ug m-3)
ndata[ndata[:, 12] > 400, 12] = np.nan
# O3
ndata[:, 14] = ndata[:, 14] / 1.96
import numpy as np
import pandas as pd
import glob
### Setting path
#data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
#path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
#mat = matlab.loadmat(os.path.join(path_station,'Station_CN', f'cn_stn_GOCI6km_location_weight.mat')) # period_GOCI.csv 사용해서 만든거
#data_base_dir = os.path.join('/data2', 'sehyun', 'Data')
data_base_dir = os.path.join('/', 'share', 'irisnas5', 'GEMS', 'GEMS_python')
#data_base_dir = os.path.join('//','10.72.26.56','irisnas5', 'GEMS', 'GEMS_python')
path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station')
mat = matlab.loadmat(
os.path.join(path_station, 'Station_CN',
f'cn_stn_GOCI6km_location_weight.mat'))
cn_dup_scode2_GOCI6km, cn_stn_GOCI6km_location = mat[
'cn_dup_scode2_GOCI6km'], mat['cn_stn_GOCI6km_location']
del mat
dup_scode2 = cn_dup_scode2_GOCI6km[:, 1:]
unq_scode2 = cn_stn_GOCI6km_location[cn_stn_GOCI6km_location[:, 8] == 0, 1]
idx = [val in dup_scode2 for val in cn_stn_GOCI6km_location[:, 1]]
dup_dist = cn_stn_GOCI6km_location[idx][:, [1, 7]]
YEARS = [2016] # range(2015, 2019+1)
for yr in YEARS:
fname = f'cn_stn_scode_data_rm_outlier_{yr}.mat'
ndata_scode = matlab.loadmat(
os.path.join(path_station, 'Station_CN/stn_scode_data/',
