def read_and_mask(yr): if os.path.isfile(os.path.join(path_read, f'{pname}_{yr}_DU.mat')): data_yr = matlab.loadmat( os.path.join(path_read, f'{pname}_{yr}_DU.mat'))['data_yr'] elif os.path.isfile(os.path.join(path_read, f'{pname}_{yr}.mat')): data_yr = matlab.loadmat( os.path.join(path_read, f'{pname}_{yr}.mat'))['data_yr'] else: raise ValueError( f'{pname}_{yr}.mat does not exists though variation tried.') data_subset = data_yr[mask == 1, :] return data_subset
def do_mask_nan_utc(fname, key): read_name = os.path.join(path_goci_aod, fname) write_name = os.path.join(path_goci_filter, fname) arr = matlab.loadmat(read_name)[key] arr[nan_utc] = np.nan matlab.savemat(fname=write_name, data={key: arr})
def read_table_mat(fname): # should contain header column mat = matlab.loadmat(fname) tmp_cols = ['KST', 'scode', 'doy', list(mat.keys())[-2]] df = pd.DataFrame(columns=tmp_cols) for col in tmp_cols: df[col] = np.squeeze(mat[col]) data = df.set_index(['KST', 'scode', 'doy']) del df print(os.path.basename(fname)) return data
import numpy as np from scipy.interpolate import griddata import time ### Setting path data_base_dir = os.path.join('/data2', 'sehyun', 'Data') path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_emis_processed = os.path.join(data_base_dir, 'Preprocessed_raw', 'EMIS') path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km') path_ea_goci_emis = os.path.join(path_ea_goci, 'EMIS') KNU_dir = 'KNU_27_01' path_read = os.path.join(path_emis_processed, KNU_dir) #%% 27 km domain mat = matlab.loadmat(os.path.join(path_grid_raw,'grid_cmaq_27km.mat')) # % lon_cmaq_27km, lat_cmaq_27km points = np.array([mat['lon_cmaq_27km'].ravel(order='F'), mat['lat_cmaq_27km'].ravel(order='F')]).T del mat print (f'points shape : {points.shape}') mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat')) # lon_goci, lat_goci lon_goci = mat['lon_goci'] lat_goci = mat['lat_goci'] del mat nr_goci, nc_goci = lon_goci.shape YEARS = [2016] for yr in YEARS: #:2016 if yr%4==0: days = 366 else: days = 365 for doy in range(1, days+1): # for doy=294:300 %1:days
if yr == 2019: days = 148 list_aod = glob.glob(os.path.join(path_goci_aod, 'AOD', str(yr), '*.mat')) list_aod = [os.path.basename(f) for f in list_aod] list_aod.sort() list_date = [x[9:20] for x in list_aod] for doy in range(1, days + 1): nan_filter = np.zeros((473, 463, 8, 4)) data = np.full([473, 463, 8], np.nan) for utc in range(num_utc): tStart = time.time() fname = os.path.join(path_goci_aod, 'AOD', str(yr), f'GOCI_AOD_{yr}_{doy:03d}_{utc:02d}.mat') GOCI_aod = matlab.loadmat(fname)['GOCI_aod'] data[:, :, utc] = GOCI_aod fname = os.path.join( path_goci_aod, 'No_of_Used_500m_Pixels_for_One_6km_Product_Pixel', str(yr), f'GOCI_num_used_pixels_{yr}_{doy:03d}_{utc:02d}.mat') GOCI_num_used_pixels = matlab.loadmat( fname)['GOCI_num_used_pixels'] GOCI_aod_bfr = np.full([477, 467], np.nan) # buffer for 5x5 moving windows GOCI_aod_bfr[2:-2, 2:-2] = GOCI_aod nan_filter = moving_window_2d(utc, GOCI_aod_bfr, nan_filter)
data_base_dir = os.path.join('/data2', 'sehyun', 'Data') path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km') path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') path_stn_jp = os.path.join(path_station, 'Station_JP') path_stn_cn = os.path.join(path_station, 'Station_CN') path_stn_kr = os.path.join(path_station, 'Station_KR') path_output = os.path.join(data_base_dir, 'output', 'RealTimeTraining', 'EastAsia') matlab.check_make_dir(path_output) tg = ['PM10','PM25'] ## Load grid mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat')) lon_goci, lat_goci = mat['lon_goci'], mat['lat_goci'] del mat ## YEARS = [2016] for yr in YEARS: #mat = matlab.loadmat(os.path.join(path_stn_kr, 'stn_GOCI6km_location_weight_v2018.mat')) mat = matlab.loadmat(os.path.join(path_stn_kr, 'stn_GOCI6km_location_weight_v201904.mat')) dup_scode2_GOCI6km = mat['dup_scode2_GOCI6km'] df = pd.DataFrame(mat['stn_GOCI6km_location'], columns=mat['header_stn_GOCI6km_location']) stn_GOCI6km_location = df.values del df, mat mat = matlab.loadmat(os.path.join(path_stn_cn, 'cn_stn_GOCI6km_location_weight.mat')) cn_dup_scode2_GOCI6km = mat['cn_dup_scode2_GOCI6km']
from Code.utils import matlab import numpy as np import rasterio as rio from scipy.ndimage import map_coordinates import time ### Setting path data_base_dir = os.path.join('/data2', 'sehyun', 'Data') path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_mcd_processed = os.path.join(data_base_dir, 'Preprocessed_raw', 'MODIS', 'MCD12Q1') path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km') path_ea_goci_modis = os.path.join(path_ea_goci, 'MODIS_LC_ratio') mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_GCS_MODIS_500m_EA.mat')) lon_gcs_500m, lat_gcs_500m = mat['lon_gcs_500m'], mat['lat_gcs_500m'] del mat dx = 0.00511542 # point x 간격 np.diff(mat['lon_gcs_500m']) dy = 0.00511542 # point y 간격 np.diff(mat['lat_gcs_500m']) xmin = np.min(lon_gcs_500m[0]) ymin = np.min(lat_gcs_500m[:, 0]) mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat')) # lon_goci, lat_goci lon_goci = mat['lon_goci'] lat_goci = mat['lat_goci'] del mat coords = ((lon_goci - xmin) / dx, (lat_goci - ymin) / dy) # points를 아래와 같이 배치
import pandas as pd import glob import h5py ### Setting path #data_base_dir = os.path.join('/data2', 'sehyun', 'Data') #path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') #data_base_dir = os.path.join('/data2', 'sehyun', 'Data') data_base_dir = os.path.join('/', 'share', 'irisnas5', 'GEMS', 'GEMS_python') #data_base_dir = os.path.join('//','10.72.26.56','irisnas5', 'GEMS', 'GEMS_python') #path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_grid_raw = os.path.join('/', 'share', 'irisnas5', 'Data', 'grid') #path_grid_raw = os.path.join('//','10.72.26.56','irisnas5','Data','grid') path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat')) latlon_data = np.array( [mat['lat_goci'].ravel(order='F'), mat['lon_goci'].ravel(order='F')]).T del mat ## China # scode1, scode2, lon, lat, op_start, op_ stn_info_cn = pd.read_csv( os.path.join(path_station, 'Station_CN', 'cn_stn_code_lonlat_period_GOCI.csv')) stn_info_cn = stn_info_cn.values new_station = np.zeros((stn_info_cn.shape[0], 4)) for i in range(stn_info_cn.shape[0]): dist = copy.deepcopy(latlon_data) dist[:, 0] = dist[:, 0] - stn_info_cn[i, 3]
import netCDF4 import numpy as np import glob from scipy.interpolate import griddata import time ### Setting path data_base_dir = os.path.join('/data2', 'sehyun', 'Data') path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_bess_raw = os.path.join(data_base_dir, 'Raw', 'BESS') # Read path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km') path_ea_goci_bess = os.path.join(path_ea_goci, 'Preprocessed_raw', 'EA_GOCI6km', 'BESS') # Write mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_bess.mat')) # lon_bess, lat_bess points = np.array( [mat['lon_bess'].ravel(order='F'), mat['lat_bess'].ravel(order='F')]).T del mat print(f'points shape : {points.shape}') mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat')) # lon_goci, lat_goci lon_goci = mat['lon_goci'] lat_goci = mat['lat_goci'] del mat YEARS = [2016] for yr in YEARS: file_list = glob.glob(os.path.join(path_bess_raw, str(yr), '*.nc')) file_list.sort()
#data_path = os.path.join(project_data, 'Data', 'Aerosol') data_base_dir = os.path.join('/', 'share', 'irisnas5', 'GEMS', 'GEMS_python') # revised rdaps_path = os.path.join(data_base_dir, 'Preprocessed_raw', 'RDAPS') # revised ### Setting period YEARS = [2016] #, 2018, 2019 for yr in YEARS: curr_path = os.path.join(path_rdaps_processed, str(yr)) if yr % 4 == 0: days = 366 else: days = 365 for i in range(1, days + 1): data00 = matlab.loadmat( os.path.join(curr_path, f'RDAPS_{yr}_{i:03d}_00.mat')) # rdaps data00 = data00['rdaps'] data06 = matlab.loadmat( os.path.join(curr_path, f'RDAPS_{yr}_{i:03d}_06.mat')) # rdaps data06 = data06['rdaps'] data12 = matlab.loadmat( os.path.join(curr_path, f'RDAPS_{yr}_{i:03d}_12.mat')) # rdaps data12 = data12['rdaps'] data18 = matlab.loadmat( os.path.join(curr_path, f'RDAPS_{yr}_{i:03d}_18.mat')) # rdaps data18 = data18['rdaps'] if i == days:
### Setting path data_base_dir = os.path.join('/data2', 'sehyun', 'Data') path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km') path_nox_korea = os.path.join(data_base_dir, 'Preprocessed_raw', 'NOX03', 'Korea') matlab.check_make_dir(path_nox_korea) path_korea_cases = os.path.join(data_base_dir, 'Preprocessed_raw', 'Korea', 'cases') path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') path_stn_jp = os.path.join(path_station, 'Station_JP') path_stn_cn = os.path.join(path_station, 'Station_CN') path_stn_kr = os.path.join(path_station, 'Station_KR') ## Load grid mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_korea.mat')) lat_kor, lon_kor = mat['lat_kor'], mat['lon_kor'] del mat path_data = '/share/irisnas6/Data/Aerosol/' path = '/share/irisnas6/Work/NOXO3/Korea/' #m_bnd = m_shaperead([path_nox_korea,'boundary/BND_SIDO_GCS']) shape_feature = ShapelyFeature(Reader(os.path.join(path_nox_korea, 'boundary', 'BND_SIDO_GCS.shp')).geometries(), ccrs.PlateCarree(), edgecolor='black') # sel_NO2 = [1,3:4,7:17,21:24,28:49,53] # sel_O3 = [1:23,25,28:49,53] #yr=2015 yr = 2016
import glob import numpy as np from scipy.interpolate import griddata import time ### Setting path data_base_dir = os.path.join('/data2', 'sehyun', 'Data') path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_gpm_processed = os.path.join(data_base_dir, 'Preprocessed_raw', 'GPM', 'AP_24h_hourly') # read path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km') path_ea_goci_gpm = os.path.join(path_ea_goci, 'Preprocessed_raw', 'EA_GOCI6km', 'GPM_AP') # Write mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat')) lat_goci, lon_goci = mat['lat_goci'], mat['lon_goci'] del mat mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_gpm.mat')) lat_gpm, lon_gpm = mat['lat_gpm'], mat['lon_gpm'] del mat lat_gpm = lat_gpm[1100:1400, 2900:3300] # N50 W110 S20 E150 lon_gpm = lon_gpm[1100:1400, 2900:3300] points = np.array([lon_gpm.ravel(order='F'), lat_gpm.ravel(order='F')]).T del lon_gpm, lat_gpm print(f'points shape : {points.shape}') YEARS = [2016] for yr in YEARS: flist = glob.glob(os.path.join(path_gpm_processed, str(yr), '*.mat'))
## header # station_header header = [ 'doy', 'yr', 'mon', 'day', 'KST', 'SO2', 'CO', 'OX', 'NO2', 'PM10', 'PM25', 'scode' ] ## YEARS = [2016] for yr in YEARS: if yr % 4 == 0: days = 366 else: days = 365 #if yr==2019: days=151; ndata = matlab.loadmat( os.path.join(path_stn_jp, 'stn_code_data', f'stn_code_data_{yr}_soramame.mat'))['stn_yr'] ndata = ndata.astype('float') scode = np.unique(ndata[:, -1]) print(ndata.shape) # SO2 ndata[:, 5] = ndata[:, 5] * 1000 # ppm to ppb ndata[ndata[:, 5] > 400, 5] = np.nan # CO ndata[:, 6] = ndata[:, 6] * 10 # ppm to 0.1ppm ndata[ndata[:, 6] > 200, 6] = np.nan # OX ndata[:, 7] = ndata[:, 7] * 1000 # ppm to ppb
base_dir = os.environ['GEMS_HOME'] project_path = os.path.join(base_dir, 'python-refactor') sys.path.insert(0, project_path) from Code.utils import matlab import time import numpy as np import rasterio as rio from scipy.ndimage import map_coordinates ### Setting path data_base_dir = os.path.join('/data2', 'sehyun', 'Data') path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km') mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_SRTM_1km.mat')) lon_SRTM_1km, lat_SRTM_1km = mat['lon_SRTM_1km'], mat['lat_SRTM_1km'] del mat dx = 0.01023084 # point x 간격 np.diff(mat['lon_gcs_500m']) dy = 0.01023084 # point y 간격 np.diff(mat['lat_gcs_500m']) xmin = np.min(lon_SRTM_1km[0]) ymin = np.min(lat_SRTM_1km[:, 0]) mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat')) # lon_goci, lat_goci lon_goci, lat_goci = mat['lon_goci'], mat['lat_goci'] del mat coords = ((lon_goci - xmin) / dx, (lat_goci - ymin) / dy) # SRTM DEM with rio.open(os.path.join(path_grid_raw, 'SRTM_DEM_1km.tif')) as src:
path_in_situ = os.path.join(data_base_dir, 'Raw') path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') path_stn_kor = os.path.join(path_station, 'Station_KR') stn_info = pd.read_csv(os.path.join(path_stn_kor, 'stn_code_lonlat_period_2005_201904.csv')) stn_info = stn_info.values # scode1, scode2, lon, lat, op_start, op_ ## read raw files header_ndata = np.array(['doy','yr','mon','day','KST','SO2','CO','O3','NO2','PM10','PM25','scode','scode2'], dtype=h5py.string_dtype(encoding='utf-8')) ## stn_scode_data for South Korea YEARS = [2016] for yr in YEARS: if yr==2019: ndata=matlab.loadmat(os.path.join(path_stn_kor,'stn_code_data', 'stn_code_data_2019_010100_042300.mat'))['ndata'] else: ndata=matlab.loadmat(os.path.join(path_stn_kor,'stn_code_data', f'stn_code_data_{yr}.mat'))['ndata'] ndata = np.hstack([ndata, np.zeros([ndata.shape[0], 1])]) # add column for scode2 ndata[ndata<0] = np.nan ndata_scode = None # Assign scode2 for j in range(stn_info.shape[0]): ndata_temp = ndata[ndata[:,11]==stn_info[j,0],:] for k in range(1,12+1): ndata_temp2 = ndata_temp[ndata_temp[:,2]==k,:] if len(ndata_temp2)!=0: yrmon = yr*100+k idx = (stn_info[j,4]<=yrmon)&(stn_info[j,5]>yrmon) if idx:
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km') path_korea_cases = os.path.join(data_base_dir, 'Preprocessed_raw', 'Korea', 'cases') matlab.check_make_dir(path_korea_cases) path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') path_stn_jp = os.path.join(path_station, 'Station_JP') path_stn_cn = os.path.join(path_station, 'Station_CN') path_stn_kr = os.path.join(path_station, 'Station_KR') tg = ['PM10', 'PM25'] ## Load grid mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_korea.mat')) lat_kor, lon_kor = mat['lat_kor'], mat['lon_kor'] del mat mat = matlab.loadmat( os.path.join(path_stn_kr, 'stn_1km_location_weight_v201904.mat')) # stn_1km_location stn_1km_location = mat['stn_1km_location'] del mat # scode1,scode2, lat_org, lon_org, pxid, lat_px, lon_px stn_location = stn_1km_location # scode1,scode2, lat_org, lon_org, pxid, lat_px, lon_px ## YEARS = [2016]
sys.path.insert(0, project_path) from Code.utils import matlab import time import numpy as np from scipy.interpolate import griddata ### Setting path data_base_dir = os.path.join('/data2', 'sehyun', 'Data') path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_omi_processed = os.path.join(data_base_dir, 'Preprocessed_raw', 'OMI') # Read path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km') path_ea_goci_omi = os.path.join(path_ea_goci, 'OMI_tempConv') # Write mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat')) # lon_goci, lat_goci lon_goci, lat_goci = mat['lon_goci'], mat['lat_goci'] del mat mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_omi_25.mat')) lon_omi, lat_omi = mat['lon_omi'], mat['lat_omi'] del mat lon_omi = lon_omi[340:560, 1020:1320] lat_omi = lat_omi[340:560, 1020, 1320] points = np.array([lon_omi.ravel(order='F'), lat_omi.ravel(order='F')]).T del lon_omi, lat_omi print(f'points shape : {points.shape}') dlist = list(
doy_000 = matlab.datenum(f'{yr}00000') doy = np.array([val - doy_000 for val in data_datenum]) stn_value = stn_tbl_temp[:, 2:15] data = np.hstack([ doy.reshape(-1, 1), dvec, stn_value[:, [0, 1, 5, 6, 3, 10, 11]], scode.reshape(-1, 1) ]) # {'doy','yr','mon','day','KST','SO2','CO','OX','NO2','PM10','PM25','scode'} if stn_mm is None: stn_mm = data else: stn_mm = np.vstack([stn_mm, data]) matlab.savemat( os.path.join(path_stn_jp, 'stn_code_data', f'stn_code_data_{yr}_{mm:02d}.mat'), {'stn_mm': stn_mm}) print(mm) print('Stack monthly data to yearly data') stn_yr = None for mm in range(1, 12 + 1): stn_mm = matlab.loadmat( os.path.join(path_stn_jp, 'stn_code_data', f'stn_code_data_{yr}_{mm:02d}.mat'))['stn_mm'] if stn_yr is None: stn_yr = stn_mm else: stn_yr = np.vstack([stn_yr, stn_mm]) matlab.savemat( os.path.join(path_stn_jp, 'stn_code_data', f'stn_code_data_{yr}_soramame.mat'), {'stn_yr': stn_yr}) print(yr)
import pandas as pd ### Setting path data_base_dir = os.path.join('/data2', 'sehyun', 'Data') path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km') path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') path_stn_jp = os.path.join(path_station, 'Station_JP') path_stn_cn = os.path.join(path_station, 'Station_CN') path_stn_kr = os.path.join(path_station, 'Station_KR') tg = ['PM10', 'PM25'] ## Load grid mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat')) lon_goci, lat_goci = mat['lon_goci'], mat['lat_goci'] del mat YEARS = [2016] for yr in YEARS: #mat = matlab.loadmat(os.path.join(path_stn_kr, 'stn_GOCI6km_location_weight_v2018.mat')) mat = matlab.loadmat( os.path.join(path_stn_kr, 'stn_GOCI6km_location_weight_v201904.mat')) dup_scode2_GOCI6km = mat['dup_scode2_GOCI6km'] df = pd.DataFrame(mat['stn_GOCI6km_location'], columns=mat['header_stn_GOCI6km_location']) stn_GOCI6km_location = df.values del df, mat mat = matlab.loadmat(
'cases') path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') path_stn_jp = os.path.join(path_station, 'Station_JP') path_stn_cn = os.path.join(path_station, 'Station_CN') path_stn_kr = os.path.join(path_station, 'Station_KR') path_data = '/share/irisnas5/Data/' path = '/share/irisnas5/GEMS/PM/00_EA6km/' season_name = ['Spring', 'Summer', 'Fall', 'Winter'] season_name2 = ['MAM', 'JJA', 'SON', 'DJF'] type_list = ['conc', 'time', 'time_conc'] target = ['PM10', 'PM25'] ## Load grid mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_korea.mat')) lat_kor, lon_kor = mat['lat_kor'], mat['lon_kor'] del mat season_num = [61, 153, 245, 336] YEARS = [2016] for t in [0, 2]: #1:2:3 for i in [0, 1]: for yr in YEARS: #2015:2017 if yr % 4 == 0: days = 366 daysInMonths = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] else: days = 365 daysInMonths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] fname = f"{target[i]}_RTT_EA6km_{yr}.mat"
path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km') path_korea_cases = os.path.join(data_base_dir, 'Preprocessed_raw', 'Korea', 'cases') matlab.check_make_dir(path_korea_cases) path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') path_stn_jp = os.path.join(path_station, 'Station_JP') path_stn_cn = os.path.join(path_station, 'Station_CN') path_stn_kr = os.path.join(path_station, 'Station_KR') tg = ['PM10', 'PM25'] ## Load grid mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_korea.mat')) lat_kor, lon_kor = mat['lat_kor'], mat['lon_kor'] del mat mat = matlab.loadmat( os.path.join(path_stn_kr, 'stn_1km_location_weight_v201904.mat')) # stn_1km_location stn_1km_location = mat['stn_1km_location'] del mat # scode1,scode2, lat_org, lon_org, pxid, lat_px, lon_px ## YEARS = [2016] for yr in YEARS: # nanidx for cases mat = matlab.loadmat(
sys.path.insert(0, project_path) from Code.utils import matlab import copy import numpy as np import pandas as pd import glob import time ### Setting path data_base_dir = os.path.join('/data2', 'sehyun', 'Data') path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') path_stn_kor = os.path.join(path_station, 'Station_KR') ## South Korea mat = matlab.loadmat( os.path.join(path_stn_kor, 'stn_GOCI6km_location_weight_v201904.mat')) dup_scode2_GOCI6km, stn_GOCI6km_location = mat['dup_scode2_GOCI6km'], mat[ 'stn_GOCI6km_location'] del mat dup_scode2 = dup_scode2_GOCI6km[:, 1:] unq_scode2 = stn_GOCI6km_location[stn_GOCI6km_location[:, 8] == 0, 1] idx = [val in dup_scode2 for val in stn_GOCI6km_location[:, 1]] dup_dist = stn_GOCI6km_location[idx][:, [1, 7]] YEARS = [2016] #2005:2019 for yr in YEARS: tStart = time.time() ndata_scode = matlab.loadmat( os.path.join(path_stn_kor, 'stn_scode_data', f'stn_scode_data_{yr}.mat'))['ndata_scode']
import random ### Setting path data_base_dir = os.path.join('/data2', 'sehyun', 'Data') path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km') path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') path_stn_jp = os.path.join(path_station, 'Station_JP') path_stn_cn = os.path.join(path_station, 'Station_CN') path_stn_kr = os.path.join(path_station, 'Station_KR') path_rtt = os.path.join(data_base_dir, 'Preprocessed_raw', 'RTT') # path_save ## Station index mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_korea.mat')) lat_kor, lon_kor = mat['lat_kor'], mat['lon_kor'] del mat mat = matlab.loadmat(os.path.join(path_stn_kr, 'stn_1km_location_weight_v201904.mat')) dup_scode2_1km = mat['dup_scode2_1km'] df = pd.DataFrame(mat['stn_1km_location'], columns=mat['header_stn_1km_location'][0]) stn_1km_location = df.values del df, mat ## Read data target = ['PM10','PM25'] # cd([path_nas4,'cases/RTT/']) i=2 #1:2 # target ######################################## header = ['NDVI','AOD','AE','FMF','SSA','RSDN','Precip','DEM','LC_ratio', # satellite data(9)
path_korea_cases = os.path.join(data_base_dir, 'Preprocessed_raw', 'Korea', 'cases') path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') path_stn_jp = os.path.join(path_station, 'Station_JP') path_stn_cn = os.path.join(path_station, 'Station_CN') path_stn_kr = os.path.join(path_station, 'Station_KR') path_data = '/share/irisnas5/Data/' path= '/share/irisnas5/GEMS/PM/00_EA6km/' target = ['PM10','PM25'] type_list = ['conc','time','time_conc'] YEARS = [2016] ## Load grid mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_korea.mat')) lat_kor, lon_kor = mat['lat_kor'], mat['lon_kor'] del mat for t in [1]: for i in [0]: for yr in YEARS: if yr%4==0: days = 366 daysInMonths = [31,29,31,30,31,30,31,31,30,31,30,31] else: days = 365 daysInMonths = [31,28,31,30,31,30,31,31,30,31,30,31] daily = [] monthly=[] annual =[]
data_base_dir = os.path.join('/', 'share', 'irisnas5', 'GEMS', 'GEMS_python') #data_base_dir = os.path.join('//', '10.72.26.56','irisnas5', 'GEMS', 'GEMS_python') path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') # scode1, scode2, lon, lat, op_start, op_ stn_info_cn = pd.read_csv(os.path.join(path_station, 'Station_CN', 'cn_stn_code_lonlat_period.csv')) stn_info_cn = stn_info_cn.values ## stn_scode_data for China header_ndata = np.array(['doy','yr','mon','day','CST','AQI','PM25','PM25_24h', 'PM10','PM10_24h','SO2','SO2_24h','NO2','NO2_24h','O3','O3_24h', 'O3_8h','O3_8h_24h','CO','CO_24h','scode','scode2'], dtype=h5py.string_dtype(encoding='utf-8')) YEARS = [2016] # range(2015, 2019+1) for yr in YEARS: fname = f'stn_code_data_rm_outlier_{yr}.mat' ndata = matlab.loadmat(os.path.join(path_station,'Station_CN', 'stn_code_data', fname))['stn_CN'] ndata = np.hstack([ndata, np.zeros([ndata.shape[0], 1])]) ndata_scode = None # Assign scode2 for j in range(stn_info_cn.shape[0]): ndata_temp = ndata[ndata[:,-2]==stn_info_cn[j,0],:] if yr==2019: mm = 5 else: mm = 12 for k in range(1,mm+1): ndata_temp2 = ndata_temp[ndata_temp[:,2]==k,:] if len(ndata_temp2)!=0: yrmon = yr*100+k
sys.path.insert(0, project_path) from Code.utils import matlab import glob import numpy as np from scipy.interpolate import griddata import time ### Setting path data_base_dir = os.path.join('/data2', 'sehyun', 'Data') path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_rdaps_processed = os.path.join(data_base_dir, 'Preprocessed_raw', 'RDAPS') path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km') path_ea_goci_rdaps = os.path.join(path_ea_goci, 'RDAPS') mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat')) # lon_goci, lat_goci lon_goci, lat_goci = mat['lon_goci'], mat['lat_goci'] del mat mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_rdaps.mat')) points = np.array( [mat['lon_rdaps'].ravel(order='F'), mat['lat_rdaps'].ravel(order='F')]).T del mat print(f'points shape : {points.shape}') YEARS = [2016] for yr in YEARS: flist = glob.glob(os.path.join(path_rdaps_processed, str(yr), '*.mat')) flist.sort() for fname in flist: # in range(1, 2000+1): tStart = time.time()
import copy import numpy as np import pandas as pd import glob import time ### Setting path #data_base_dir = os.path.join('/data2', 'sehyun', 'Data') #data_base_dir = os.path.join('//', '10.72.26.56','irisnas5', 'GEMS', 'GEMS_python') data_base_dir = os.path.join('/', 'share', 'irisnas5', 'GEMS', 'GEMS_python') path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') path_stn_jp = os.path.join(path_station, 'Station_JP') ## Japan mat = matlab.loadmat( os.path.join(path_stn_jp, 'jp_stn_GOCI6km_location_weight_v2017.mat')) jp_dup_scode2_GOCI6km, jp_stn_GOCI6km_location = mat[ 'jp_dup_scode2_GOCI6km'], mat['jp_stn_GOCI6km_location'] del mat dup_scode2 = jp_dup_scode2_GOCI6km[:, 1:] unq_scode2 = jp_stn_GOCI6km_location[jp_stn_GOCI6km_location[:, 8] == 0, 1] idx = [val in dup_scode2 for val in jp_stn_GOCI6km_location[:, 1]] dup_dist = jp_stn_GOCI6km_location[idx][:, [1, 7]] YEARS = [2016] for yr in YEARS: tStart = time.time() if os.path.isfile( os.path.join(path_stn_jp, 'stn_scode_data', f'jp_stn_scode_data_{yr}.mat')):
import random ### Setting path data_base_dir = os.path.join('/data2', 'sehyun', 'Data') path_grid_raw = os.path.join(data_base_dir, 'Raw', 'grid') path_ea_goci = os.path.join(data_base_dir, 'Preprocessed_raw', 'EA_GOCI6km') path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') path_stn_jp = os.path.join(path_station, 'Station_JP') path_stn_cn = os.path.join(path_station, 'Station_CN') path_stn_kr = os.path.join(path_station, 'Station_KR') path_rtt = os.path.join(data_base_dir, 'Preprocessed_raw', 'RTT') # path_save ## Station index mat = matlab.loadmat(os.path.join(path_grid_raw, 'grid_goci.mat')) lon_goci, lat_goci = mat['lon_goci'], mat['lat_goci'] del mat mat = matlab.loadmat( os.path.join(path_stn_cn, 'cn_stn_GOCI6km_location_weight.mat')) cn_dup_scode2_GOCI6km = mat['cn_dup_scode2_GOCI6km'] header_cn_stn_GOCI6km_location = mat['header_cn_stn_GOCI6km_location'] df = pd.DataFrame(mat['cn_stn_GOCI6km_location'], columns=header_cn_stn_GOCI6km_location) cn_stn_GOCI6km_location = df.values del df, mat #mat = matlab.loadmat(os.path.join(path_stn_kr, 'stn_GOCI6km_location_weight_v2018.mat')) mat = matlab.loadmat( os.path.join(path_stn_kr, 'stn_GOCI6km_location_weight_v201904.mat'))
#data_base_dir = os.path.join('/data2', 'sehyun', 'Data') #path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') #data_base_dir = os.path.join('/data2', 'sehyun', 'Data') #path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') data_base_dir = os.path.join('/', 'share', 'irisnas5', 'GEMS', 'GEMS_python') path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') YEARS = [2016] # range(2015, 2019+1) for yr in YEARS: if yr % 4 == 0: days = 366 else: days = 365 if yr == 2019: days = 151 ndata = matlab.loadmat( os.path.join(path_station, 'Station_CN', 'stn_code_data', f'stn_code_data_{yr}.mat'))['stn_yr'] scode = np.unique(ndata[:, -1]) # CO ndata[:, 18] = ndata[:, 18] / 1.15 # (mg/m3) to ppm (1 ppm = 1.15 mg m-3) ndata[ndata[:, 18] > 20, 18] = np.nan # SO2 ndata[:, 10] = ndata[:, 10] / 2.62 # (ug/m3) to ppb (1 ppb = 2.62 ug m-3) ndata[ndata[:, 10] > 400, 10] = np.nan # NO2 ndata[:, 12] = ndata[:, 12] / 1.88 # (ug/m3) to ppb (1 ppb = 1.88 ug m-3) ndata[ndata[:, 12] > 400, 12] = np.nan # O3 ndata[:, 14] = ndata[:, 14] / 1.96
import numpy as np import pandas as pd import glob ### Setting path #data_base_dir = os.path.join('/data2', 'sehyun', 'Data') #path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') #mat = matlab.loadmat(os.path.join(path_station,'Station_CN', f'cn_stn_GOCI6km_location_weight.mat')) # period_GOCI.csv 사용해서 만든거 #data_base_dir = os.path.join('/data2', 'sehyun', 'Data') data_base_dir = os.path.join('/', 'share', 'irisnas5', 'GEMS', 'GEMS_python') #data_base_dir = os.path.join('//','10.72.26.56','irisnas5', 'GEMS', 'GEMS_python') path_station = os.path.join(data_base_dir, 'Preprocessed_raw', 'Station') mat = matlab.loadmat( os.path.join(path_station, 'Station_CN', f'cn_stn_GOCI6km_location_weight.mat')) cn_dup_scode2_GOCI6km, cn_stn_GOCI6km_location = mat[ 'cn_dup_scode2_GOCI6km'], mat['cn_stn_GOCI6km_location'] del mat dup_scode2 = cn_dup_scode2_GOCI6km[:, 1:] unq_scode2 = cn_stn_GOCI6km_location[cn_stn_GOCI6km_location[:, 8] == 0, 1] idx = [val in dup_scode2 for val in cn_stn_GOCI6km_location[:, 1]] dup_dist = cn_stn_GOCI6km_location[idx][:, [1, 7]] YEARS = [2016] # range(2015, 2019+1) for yr in YEARS: fname = f'cn_stn_scode_data_rm_outlier_{yr}.mat' ndata_scode = matlab.loadmat( os.path.join(path_station, 'Station_CN/stn_scode_data/',