def __init__(self, CONFIG, period, region, passwd):
self.STDMET_CONFIG = CONFIG['stdmet']
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.years = [x for x in range(self.period[0].year,
self.period[1].year+1)]
self.stations = set()
self.logger = logging.getLogger(__name__)
self._analysis_and_save_relation()
self._extract_relation()
# self.download()
# self.download_all_stations_no_limit()
utils.reset_signal_handler()
utils.setup_database(self, Base)
# self.read()
self.show()
def __init__(self, CONFIG, period, region, passwd):
self.SFMR_CONFIG = CONFIG['sfmr']
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.logger = logging.getLogger(__name__)
utils.setup_database(self, Base)
self.years = [
x for x in range(self.period[0].year, self.period[1].year + 1)
]
brief_info = self.get_all_hurricanes_brief_info()
self.download_and_update_by_brief_info(brief_info)
return
self._gen_all_year_hurr()
self._extract_year_hurr()
self.download()
utils.reset_signal_handler()
read_all = False
self._extract_year_hurr_file_path(read_all)
def __init__(self, CONFIG, period, region, passwd):
self.logger = logging.getLogger(__name__)
self.satel_names = ['ascat', 'wsat', 'amsr2', 'smap', 'sentinel_1']
self.CONFIG = CONFIG
self.period = period
self.region = region
self.lat1, self.lat2 = region[0], region[1]
self.lon1, self.lon2 = region[2], region[3]
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.edge = self.CONFIG['regression']['edge_in_degree']
self.spa_resolu = self.CONFIG['rss']['spatial_resolution']
self.grid_pts = dict()
self.grid_pts['rss'] = dict()
self.grid_pts['rss']['lat'] = [
y * self.spa_resolu - 89.875
for y in range(self.CONFIG['rss']['lat_grid_points_number'])
]
self.grid_pts['rss']['lon'] = [
x * self.spa_resolu + 0.125
for x in range(self.CONFIG['rss']['lon_grid_points_number'])
]
self.grid_pts['era5'] = dict()
self.grid_pts['era5']['lat'] = [
y * self.spa_resolu - 90
for y in range(self.CONFIG['era5']['lat_grid_points_number'])
]
self.grid_pts['era5']['lon'] = [
x * self.spa_resolu
for x in range(self.CONFIG['era5']['lon_grid_points_number'])
]
self.missing_value = dict()
for satel_name in self.satel_names:
if satel_name != 'smap':
self.missing_value[satel_name] = \
self.CONFIG[satel_name]['missing_value']
else:
self.missing_value[satel_name] = dict()
self.missing_value[satel_name]['minute'] = \
self.CONFIG[satel_name]['missing_value']['minute']
self.missing_value[satel_name]['wind'] = \
self.CONFIG[satel_name]['missing_value']['wind']
utils.setup_database(self, Base)
self.grid_df = pd.read_sql('SELECT * FROM grid', self.engine)
self.grid_lons = self.grid_df['lon']
self.grid_lats = self.grid_df['lat']
self.grid_x = self.grid_df['x']
self.grid_y = self.grid_df['y']
self.zorders = self.CONFIG['plot']['zorders']['scs_basemap']
self.draw_coverage()
def __init__(self, CONFIG, period, region, passwd, save_disk):
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.save_disk = save_disk
self.engine = None
self.session = None
self.logger = logging.getLogger(__name__)
utils.setup_database(self, Base)
self.years = [
x for x in range(self.period[0].year, self.period[1].year + 1)
]
self.lat1, self.lat2 = region[0], region[1]
self.lon1, self.lon2 = region[2], region[3]
utils.setup_database(self, Base)
self.grid_lons = None
self.grid_lats = None
self.grid_x = None
self.grid_y = None
# Load 4 variables above
utils.load_grid_lonlat_xy(self)
self.sources = ['era5', 'smap']
# self.compare_with_isd()
self.visualization()
def __init__(self, CONFIG, period, region, basin, passwd, save_disk, work):
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.save_disk = save_disk
self.engine = None
self.session = None
self.basin = basin
self.logger = logging.getLogger(__name__)
utils.setup_database(self, Base)
self.years = [
x for x in range(self.period[0].year, self.period[1].year + 1)
]
self.lat1, self.lat2 = region[0], region[1]
self.lon1, self.lon2 = region[2], region[3]
self.lats = dict()
self.lons = dict()
self.spa_resolu = dict()
self.spa_resolu['smap'] = self.CONFIG['rss']['spatial_resolution']
self.lats['smap'] = [
y * self.spa_resolu['smap'] - 89.875 for y in range(720)
]
self.lons['smap'] = [
x * self.spa_resolu['smap'] + 0.125 for x in range(1440)
]
self.spa_resolu['era5'] = self.CONFIG['era5']['spatial_resolution']
self.lats['era5'] = [
y * self.spa_resolu['era5'] - 90 for y in range(721)
]
self.lons['era5'] = [x * self.spa_resolu['era5'] for x in range(1440)]
self.edge = self.CONFIG['regression']['edge_in_degree']
self.half_edge = self.edge / 2
self.half_edge_grid_intervals = int(self.half_edge /
self.spa_resolu['smap'])
self.pres_lvls = self.CONFIG['era5']['pres_lvls']
utils.setup_database(self, Base)
self.grid_lons = None
self.grid_lats = None
self.grid_x = None
self.grid_y = None
# Load 4 variables above
utils.load_grid_lonlat_xy(self)
self.sources = ['era5', 'smap']
if work:
self.extract()
def __init__(self, CONFIG, train_period, test_period, region, passwd,
save_disk):
self.logger = logging.getLogger(__name__)
self.CONFIG = CONFIG
self.train_period = train_period
self.test_period = test_period
self.region = region
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.save_disk = save_disk
self.satel_names = self.CONFIG['satel_names']
self.lat1, self.lat2 = region[0], region[1]
self.lon1, self.lon2 = region[2], region[3]
self.useless_columns = self.CONFIG['regression']\
['useless_columns']
self.edge = self.CONFIG['regression']['edge_in_degree']
self.spa_resolu = self.CONFIG['era5']['spatial_resolution']
self.lat_grid_points = [
y * self.spa_resolu - 90
for y in range(self.CONFIG['era5']['lat_grid_points_number'])
]
self.lon_grid_points = [
x * self.spa_resolu
for x in range(self.CONFIG['era5']['lon_grid_points_number'])
]
self.epochs = 500
self.batch_size = 32
self.validation_split = 0.2
self.grid_lons = None
self.grid_lats = None
self.grid_x = None
self.grid_y = None
# Load 4 variables above
utils.load_grid_lonlat_xy(self)
self.predict_table_name_prefix = 'predicted_smap_tc'
self.wind_radii = self.CONFIG['wind_radii']
utils.setup_database(self, Base)
self.months = self._gen_months_in_period()
self.zorders = self.CONFIG['plot']['zorders']['scs_basemap']
for satel_name in self.satel_names:
self.logger.info((f"""Training regression model of """
f"""{satel_name} and predicting data """
f"""using ERA5 reanalysis"""))
self.read_satel_era5(satel_name)
self.make_satel_DNNs(satel_name)
self.train_satel_DNNs(satel_name)
self.predict(satel_name)
def __init__(self, CONFIG, period, basin, passwd):
self.CONFIG = CONFIG
self.period = period
self.basin = basin
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.logger = logging.getLogger(__name__)
utils.setup_database(self, Base)
self.matchup_smap_sfmr()
def __init__(self, CONFIG, train_period, train_test_split_dt,
region, basin, passwd, save_disk, instructions,
smogn_target):
self.logger = logging.getLogger(__name__)
self.CONFIG = CONFIG
self.train_period = train_period
self.train_test_split_dt = train_test_split_dt
self.region = region
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.save_disk = save_disk
self.basin = basin
self.instructions = instructions
self.smogn_target = smogn_target
utils.setup_database(self, Base)
self.y_name = 'smap_windspd'
self.model_dir = self.CONFIG["classify"]["dirs"]['lightgbm'][
'model']
self.set_variables()
self.read_smap_era5()
self.classify_threshold = 45
self.label_y()
self.lgb_train = lgb.Dataset(self.X_train, self.y_train,
free_raw_data=False)
self.lgb_eval = lgb.Dataset(self.X_test, self.y_test,
reference=self.lgb_train,
free_raw_data=False)
if 'lgb' in self.instructions:
self.record_best_params()
if 'optimize' in self.instructions:
self.optimize_classifier(maxevals=100)
if 'compare' in self.instructions:
self.compare_classifier()
# elif 'compare_best' in self.instructions:
# self.compare_lgb_best_regressor()
# elif 'best' in self.instructions:
# self.lgb_best_regressor()
# elif 'default' in self.instructions:
# self.lgb_default_regressor()
else:
self.logger.error('Nothing happen')
def __init__(self, CONFIG, period, region, basin, passwd, save_disk):
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.save_disk = save_disk
self.engine = None
self.session = None
self.basin = basin
self.logger = logging.getLogger(__name__)
utils.setup_database(self, Base)
# self.load_match()
# breakpoint()
self.extract()
def __init__(self, CONFIG, region, passwd, run):
self.logger = logging.getLogger(__name__)
self.CONFIG = CONFIG
self.region = region
self.lat1, self.lat2 = region[0], region[1]
self.lon1, self.lon2 = region[2], region[3]
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.spa_resolu = self.CONFIG['grid']['spatial_resolution']
utils.setup_database(self, Base)
if run:
self.setup_grid()
def __init__(self, CONFIG, period, region, basin, passwd):
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.basin = basin
self.logger = logging.getLogger(__name__)
utils.setup_database(self, Base)
self.years = [
x for x in range(self.period[0].year, self.period[1].year + 1)
]
self.combine()
def __init__(self, CONFIG, period, region, passwd):
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.logger = logging.getLogger(__name__)
self.years = [
x for x in range(self.period[0].year, self.period[1].year + 1)
]
utils.setup_database(self, Base)
self.download()
utils.reset_signal_handler()
self.read()
def __init__(self, CONFIG, period, region, passwd, work_mode):
self.logger = logging.getLogger(__name__)
self.CONFIG = CONFIG
self.period = period
self.db_root_passwd = passwd
self.work_mode = work_mode
self.engine = None
self.session = None
self.years = [
x for x in range(self.period[0].year, self.period[1].year + 1)
]
self.lat1, self.lat2 = region[0], region[1]
self.lon1, self.lon2 = region[2], region[3]
self.zorders = {
'coastlines': 4,
'mapboundary': 0,
'continents': 8,
'station': 9,
'contour': 3,
'contourf': 2,
'wedge': 7,
'grid': 10
}
utils.setup_database(self, Base)
# self.read_scs_stations()
self.grid_lons = None
self.grid_lats = None
self.grid_x = None
self.grid_y = None
# Load 4 variables above
utils.load_grid_lonlat_xy(self)
# self.if_station_on_land()
# self.plot_stations_on_map()
if self.work_mode != '':
self.download_and_read_scs_data()
def __init__(self, CONFIG, period, basin, sources, passwd):
self.logger = logging.getLogger(__name__)
self.CONFIG = CONFIG
self.period = period
self.basin = basin
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.logger = logging.getLogger(__name__)
utils.setup_database(self, Base)
self.period_str = []
for dt in self.period:
self.period_str.append(dt.strftime('%Y-%m-%d %H:%M:%S'))
src_name_priority = {
'smap_prediction': 1,
'smap': 2,
'era5': 3,
}
# if len(sources) != 2:
# self.logger.error('Sources length should be 2')
# exit(1)
self.sources = sorted(sources, key=src_name_priority.get)
self.src_1 = self.sources[0]
if len(sources) == 2:
self.src_2 = self.sources[1]
src_plot_names_mapper = {
'smap_prediction': 'HYBRID',
'smap': 'SMAP',
'era5': 'ERA5',
}
self.src_plot_names = [
src_plot_names_mapper[src] for src in self.sources
]
# self.show_hist_of_matchup()
# breakpoint()
# if 'smap_prediction' in self.sources:
# self.add_dist2coast()
self.compare_two_sources()
def __init__(self, CONFIG, period, basin, passwd):
self.CONFIG = CONFIG
self.period = period
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.basin = basin
self.logger = logging.getLogger(__name__)
utils.setup_database(self, Base)
# Get IBTrACS table
table_name = self.CONFIG['ibtracs']['table_name'][self.basin]
IBTrACS = utils.get_class_by_tablename(self.engine, table_name)
self.tc_query = self.session.query(IBTrACS).filter(
IBTrACS.date_time >= self.period[0],
IBTrACS.date_time <= self.period[1])
self.tc_query_num = self.tc_query.count()
self.detect_rapid_intensification()
def __init__(self, CONFIG, period, region, basin, passwd):
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.basin = basin
self.logger = logging.getLogger(__name__)
self.years = [
x for x in range(self.period[0].year, self.period[1].year + 1)
]
self.lat1, self.lat2 = self.region[0], self.region[1]
self.lon1, self.lon2 = self.region[2], self.region[3]
self.download(self.basin)
utils.reset_signal_handler()
utils.setup_database(self, Base)
self.read(self.basin, region_restriction=False)
def __init__(self, CONFIG, period, region, passwd, work, save_disk,
work_mode, vars_mode):
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.save_disk = save_disk
self.logger = logging.getLogger(__name__)
self.lat1, self.lat2 = region[0], region[1]
self.lon1, self.lon2 = region[2], region[3]
self.years = [
x for x in range(self.period[0].year, self.period[1].year + 1)
]
self.main_hours = self.CONFIG['era5']['main_hours']
self.edge = self.CONFIG['regression']['edge_in_degree']
self.cdsapi_client = cdsapi.Client()
self.vars = self.CONFIG['era5']['vars']
self.surface_pres_lvl = '1000'
self.spa_resolu = self.CONFIG['era5']['spatial_resolution']
self.lat_grid_points = [
y * self.spa_resolu - 90
for y in range(self.CONFIG['era5']['lat_grid_points_number'])
]
self.lon_grid_points = [
x * self.spa_resolu
for x in range(self.CONFIG['era5']['lon_grid_points_number'])
]
# Size of threeD grid points around TC center
self.threeD_grid = dict()
self.threeD_grid['lat_axis'] = self.threeD_grid['lon_axis'] = int(
self.edge / self.spa_resolu) + 1
self.threeD_grid['height'] = len(self.CONFIG['era5']['pres_lvls'])
self.twoD_grid = dict()
self.twoD_grid['lat_axis'] = self.twoD_grid['lon_axis'] = int(
self.edge / self.spa_resolu) + 1
self.wind_radii = self.CONFIG['wind_radii']
self.grid_pts = dict()
self.grid_pts['era5'] = dict()
self.grid_pts['era5']['lat'] = [
y * self.spa_resolu - 90
for y in range(self.CONFIG['era5']['lat_grid_points_number'])
]
self.grid_pts['era5']['lon'] = [
x * self.spa_resolu
for x in range(self.CONFIG['era5']['lon_grid_points_number'])
]
self.zorders = self.CONFIG['plot']['zorders']['compare']
utils.setup_database(self, Base)
self.grid_lons = None
self.grid_lats = None
self.grid_x = None
self.grid_y = None
# Load 4 variables above
utils.load_grid_lonlat_xy(self)
if work:
# work_mode: 'scs' / 'tc'
# vars_mode: 'surface_all_vars' / 'threeD' / 'surface_wind'
self.download_and_read(work_mode, vars_mode)
def __init__(self, CONFIG, train_period, train_test_split_dt,
region, basin, passwd, save_disk, instructions,
smogn_target, tag):
self.logger = logging.getLogger(__name__)
self.CONFIG = CONFIG
self.train_period = train_period
self.train_test_split_dt = train_test_split_dt
self.region = region
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.save_disk = save_disk
self.basin = basin
self.instructions = instructions
self.smogn_target = smogn_target
self.tag = tag
self.edge = self.CONFIG['regression']['edge_in_degree']
self.spa_resolu = self.CONFIG['era5']['spatial_resolution']
self.lat_grid_points = [y * self.spa_resolu - 90 for y in range(
self.CONFIG['era5']['lat_grid_points_number'])]
self.lon_grid_points = [x * self.spa_resolu for x in range(
self.CONFIG['era5']['lon_grid_points_number'])]
self.model_dir = dict()
self.model_dir['lgb'] = self.CONFIG['regression']['dirs'][
'tc']['lightgbm']['model']
self.validation_split = 0.2
self.predict_table_name_prefix = 'predicted_smap_tc'
self.compare_zorders = self.CONFIG['plot']['zorders'][
'compare']
self.wind_radii = self.CONFIG['wind_radii']
utils.setup_database(self, Base)
self.y_name = self.CONFIG['regression']['target']['smap_era5']
# self.y_name = self.CONFIG['regression']['target']['sfmr_era5']
self.set_variables()
self.read_smap_era5()
self.show_dataset_shape()
self.lgb_train = lgb.Dataset(self.X_train, self.y_train,
free_raw_data=False)
self.lgb_eval = lgb.Dataset(self.X_test, self.y_test,
reference=self.lgb_train,
free_raw_data=False)
if 'vote' in self.instructions:
self.voting_regressor()
if 'lgb' in self.instructions:
if 'optimize' in self.instructions:
self.optimize_lgb(maxevals=3)
elif 'compare_best' in self.instructions:
self.compare_lgb_best_regressor()
elif 'best' in self.instructions:
self.lgb_best_regressor()
elif 'default' in self.instructions:
self.lgb_default_regressor()
else:
self.logger.error('Nothing happen')
def __init__(self, CONFIG, period, region, basin, passwd, save_disk,
simulate_instructions):
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.save_disk = save_disk
self.engine = None
self.session = None
self.simulate_instructions = list(set(simulate_instructions))
self.basin = basin
self.logger = logging.getLogger(__name__)
utils.setup_database(self, Base)
self.lat1, self.lat2 = region[0], region[1]
self.lon1, self.lon2 = region[2], region[3]
self.spa_resolu = dict()
self.spa_resolu['era5'] = dict()
self.spa_resolu['era5']['atm'] = self.CONFIG['era5'][
'spatial_resolution']
self.spa_resolu['era5']['ocean'] = self.CONFIG['era5'][
'ocean_spatial_resolution']
self.lats = dict()
self.lons = dict()
self.lats['era5'] = [
y * self.spa_resolu['era5']['atm'] - 90 for y in range(721)
]
self.lons['era5'] = [
x * self.spa_resolu['era5']['atm'] for x in range(1440)
]
self.spa_resolu['smap'] = self.CONFIG['rss']['spatial_resolution']
self.lats['smap'] = [
y * self.spa_resolu['smap'] - 89.875 for y in range(720)
]
self.lons['smap'] = [
x * self.spa_resolu['smap'] + 0.125 for x in range(1440)
]
self.reg_edge = self.CONFIG['regression']['edge_in_degree']
self.half_reg_edge = self.reg_edge / 2
self.half_reg_edge_grid_intervals = int(self.half_reg_edge /
self.spa_resolu['era5']['atm'])
self.pres_lvls = self.CONFIG['era5']['pres_lvls']
self.grid_lons = None
self.grid_lats = None
self.grid_x = None
self.grid_y = None
# Load 4 variables above
utils.load_grid_lonlat_xy(self)
self.zorders = self.CONFIG['plot']['zorders']['compare']
utils.reset_signal_handler()
self.tc_names = []
for name in self.simulate_instructions:
self.tc_names.append(name.upper())
self.simulate_smap_windspd()
def __init__(self, CONFIG, period, region, passwd, work_mode):
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.logger = logging.getLogger(__name__)
utils.setup_database(self, Base)
self.lat1, self.lat2 = region[0], region[1]
self.lon1, self.lon2 = region[2], region[3]
self.spa_resolu = dict()
self.spa_resolu['ccmp'] = self.CONFIG['ccmp']\
['spatial_resolution']
self.spa_resolu['grid'] = self.CONFIG['grid']\
['spatial_resolution']
self.grid_pts = dict()
self.grid_pts['ccmp'] = dict()
self.grid_pts['ccmp']['lat'] = [
y * self.spa_resolu['ccmp'] - 78.375
for y in range(self.CONFIG['ccmp']['lat_grid_points_number'])
]
self.grid_pts['ccmp']['lon'] = [
x * self.spa_resolu['ccmp'] + 0.125
for x in range(self.CONFIG['ccmp']['lon_grid_points_number'])
]
self.grid_lons = None
self.grid_lats = None
self.grid_x = None
self.grid_y = None
# Load 4 variables above
utils.load_grid_lonlat_xy(self)
self.zorders = self.CONFIG['plot']['zorders']['scs_basemap']
self._get_region_corners_indices()
self.root_url = self.CONFIG['ccmp']['url']
self.filename_prefix = self.CONFIG['ccmp']['filename']\
['prefix']
self.filename_suffix = self.CONFIG['ccmp']['filename']\
['suffix']
self.root_dir = self.CONFIG['ccmp']['dir']
utils.set_format_custom_text(self.CONFIG['ccmp']['filename_length'])
if work_mode == 'fetch':
self.download('tc')
self.read()
elif work_mode == 'compare':
self.compare_ccmp_with_ibtracs()
elif work_mode == 'fetch_and_compare':
self.download('tc')
self.read()
self.compare_ccmp_with_ibtracs()
