def __init__(self, CONFIG, period, region, passwd): self.STDMET_CONFIG = CONFIG['stdmet'] self.CONFIG = CONFIG self.period = period self.region = region self.db_root_passwd = passwd self.engine = None self.session = None self.years = [x for x in range(self.period[0].year, self.period[1].year+1)] self.stations = set() self.logger = logging.getLogger(__name__) self._analysis_and_save_relation() self._extract_relation() # self.download() # self.download_all_stations_no_limit() utils.reset_signal_handler() utils.setup_database(self, Base) # self.read() self.show()
def __init__(self, CONFIG, period, region, passwd): self.SFMR_CONFIG = CONFIG['sfmr'] self.CONFIG = CONFIG self.period = period self.region = region self.db_root_passwd = passwd self.engine = None self.session = None self.logger = logging.getLogger(__name__) utils.setup_database(self, Base) self.years = [ x for x in range(self.period[0].year, self.period[1].year + 1) ] brief_info = self.get_all_hurricanes_brief_info() self.download_and_update_by_brief_info(brief_info) return self._gen_all_year_hurr() self._extract_year_hurr() self.download() utils.reset_signal_handler() read_all = False self._extract_year_hurr_file_path(read_all)
def __init__(self, CONFIG, period, region, passwd): self.logger = logging.getLogger(__name__) self.satel_names = ['ascat', 'wsat', 'amsr2', 'smap', 'sentinel_1'] self.CONFIG = CONFIG self.period = period self.region = region self.lat1, self.lat2 = region[0], region[1] self.lon1, self.lon2 = region[2], region[3] self.db_root_passwd = passwd self.engine = None self.session = None self.edge = self.CONFIG['regression']['edge_in_degree'] self.spa_resolu = self.CONFIG['rss']['spatial_resolution'] self.grid_pts = dict() self.grid_pts['rss'] = dict() self.grid_pts['rss']['lat'] = [ y * self.spa_resolu - 89.875 for y in range(self.CONFIG['rss']['lat_grid_points_number']) ] self.grid_pts['rss']['lon'] = [ x * self.spa_resolu + 0.125 for x in range(self.CONFIG['rss']['lon_grid_points_number']) ] self.grid_pts['era5'] = dict() self.grid_pts['era5']['lat'] = [ y * self.spa_resolu - 90 for y in range(self.CONFIG['era5']['lat_grid_points_number']) ] self.grid_pts['era5']['lon'] = [ x * self.spa_resolu for x in range(self.CONFIG['era5']['lon_grid_points_number']) ] self.missing_value = dict() for satel_name in self.satel_names: if satel_name != 'smap': self.missing_value[satel_name] = \ self.CONFIG[satel_name]['missing_value'] else: self.missing_value[satel_name] = dict() self.missing_value[satel_name]['minute'] = \ self.CONFIG[satel_name]['missing_value']['minute'] self.missing_value[satel_name]['wind'] = \ self.CONFIG[satel_name]['missing_value']['wind'] utils.setup_database(self, Base) self.grid_df = pd.read_sql('SELECT * FROM grid', self.engine) self.grid_lons = self.grid_df['lon'] self.grid_lats = self.grid_df['lat'] self.grid_x = self.grid_df['x'] self.grid_y = self.grid_df['y'] self.zorders = self.CONFIG['plot']['zorders']['scs_basemap'] self.draw_coverage()
def __init__(self, CONFIG, period, region, passwd, save_disk): self.CONFIG = CONFIG self.period = period self.region = region self.db_root_passwd = passwd self.save_disk = save_disk self.engine = None self.session = None self.logger = logging.getLogger(__name__) utils.setup_database(self, Base) self.years = [ x for x in range(self.period[0].year, self.period[1].year + 1) ] self.lat1, self.lat2 = region[0], region[1] self.lon1, self.lon2 = region[2], region[3] utils.setup_database(self, Base) self.grid_lons = None self.grid_lats = None self.grid_x = None self.grid_y = None # Load 4 variables above utils.load_grid_lonlat_xy(self) self.sources = ['era5', 'smap'] # self.compare_with_isd() self.visualization()
def __init__(self, CONFIG, period, region, basin, passwd, save_disk, work): self.CONFIG = CONFIG self.period = period self.region = region self.db_root_passwd = passwd self.save_disk = save_disk self.engine = None self.session = None self.basin = basin self.logger = logging.getLogger(__name__) utils.setup_database(self, Base) self.years = [ x for x in range(self.period[0].year, self.period[1].year + 1) ] self.lat1, self.lat2 = region[0], region[1] self.lon1, self.lon2 = region[2], region[3] self.lats = dict() self.lons = dict() self.spa_resolu = dict() self.spa_resolu['smap'] = self.CONFIG['rss']['spatial_resolution'] self.lats['smap'] = [ y * self.spa_resolu['smap'] - 89.875 for y in range(720) ] self.lons['smap'] = [ x * self.spa_resolu['smap'] + 0.125 for x in range(1440) ] self.spa_resolu['era5'] = self.CONFIG['era5']['spatial_resolution'] self.lats['era5'] = [ y * self.spa_resolu['era5'] - 90 for y in range(721) ] self.lons['era5'] = [x * self.spa_resolu['era5'] for x in range(1440)] self.edge = self.CONFIG['regression']['edge_in_degree'] self.half_edge = self.edge / 2 self.half_edge_grid_intervals = int(self.half_edge / self.spa_resolu['smap']) self.pres_lvls = self.CONFIG['era5']['pres_lvls'] utils.setup_database(self, Base) self.grid_lons = None self.grid_lats = None self.grid_x = None self.grid_y = None # Load 4 variables above utils.load_grid_lonlat_xy(self) self.sources = ['era5', 'smap'] if work: self.extract()
def __init__(self, CONFIG, train_period, test_period, region, passwd, save_disk): self.logger = logging.getLogger(__name__) self.CONFIG = CONFIG self.train_period = train_period self.test_period = test_period self.region = region self.db_root_passwd = passwd self.engine = None self.session = None self.save_disk = save_disk self.satel_names = self.CONFIG['satel_names'] self.lat1, self.lat2 = region[0], region[1] self.lon1, self.lon2 = region[2], region[3] self.useless_columns = self.CONFIG['regression']\ ['useless_columns'] self.edge = self.CONFIG['regression']['edge_in_degree'] self.spa_resolu = self.CONFIG['era5']['spatial_resolution'] self.lat_grid_points = [ y * self.spa_resolu - 90 for y in range(self.CONFIG['era5']['lat_grid_points_number']) ] self.lon_grid_points = [ x * self.spa_resolu for x in range(self.CONFIG['era5']['lon_grid_points_number']) ] self.epochs = 500 self.batch_size = 32 self.validation_split = 0.2 self.grid_lons = None self.grid_lats = None self.grid_x = None self.grid_y = None # Load 4 variables above utils.load_grid_lonlat_xy(self) self.predict_table_name_prefix = 'predicted_smap_tc' self.wind_radii = self.CONFIG['wind_radii'] utils.setup_database(self, Base) self.months = self._gen_months_in_period() self.zorders = self.CONFIG['plot']['zorders']['scs_basemap'] for satel_name in self.satel_names: self.logger.info((f"""Training regression model of """ f"""{satel_name} and predicting data """ f"""using ERA5 reanalysis""")) self.read_satel_era5(satel_name) self.make_satel_DNNs(satel_name) self.train_satel_DNNs(satel_name) self.predict(satel_name)
def __init__(self, CONFIG, period, basin, passwd): self.CONFIG = CONFIG self.period = period self.basin = basin self.db_root_passwd = passwd self.engine = None self.session = None self.logger = logging.getLogger(__name__) utils.setup_database(self, Base) self.matchup_smap_sfmr()
def __init__(self, CONFIG, train_period, train_test_split_dt, region, basin, passwd, save_disk, instructions, smogn_target): self.logger = logging.getLogger(__name__) self.CONFIG = CONFIG self.train_period = train_period self.train_test_split_dt = train_test_split_dt self.region = region self.db_root_passwd = passwd self.engine = None self.session = None self.save_disk = save_disk self.basin = basin self.instructions = instructions self.smogn_target = smogn_target utils.setup_database(self, Base) self.y_name = 'smap_windspd' self.model_dir = self.CONFIG["classify"]["dirs"]['lightgbm'][ 'model'] self.set_variables() self.read_smap_era5() self.classify_threshold = 45 self.label_y() self.lgb_train = lgb.Dataset(self.X_train, self.y_train, free_raw_data=False) self.lgb_eval = lgb.Dataset(self.X_test, self.y_test, reference=self.lgb_train, free_raw_data=False) if 'lgb' in self.instructions: self.record_best_params() if 'optimize' in self.instructions: self.optimize_classifier(maxevals=100) if 'compare' in self.instructions: self.compare_classifier() # elif 'compare_best' in self.instructions: # self.compare_lgb_best_regressor() # elif 'best' in self.instructions: # self.lgb_best_regressor() # elif 'default' in self.instructions: # self.lgb_default_regressor() else: self.logger.error('Nothing happen')
def __init__(self, CONFIG, period, region, basin, passwd, save_disk): self.CONFIG = CONFIG self.period = period self.region = region self.db_root_passwd = passwd self.save_disk = save_disk self.engine = None self.session = None self.basin = basin self.logger = logging.getLogger(__name__) utils.setup_database(self, Base) # self.load_match() # breakpoint() self.extract()
def __init__(self, CONFIG, region, passwd, run): self.logger = logging.getLogger(__name__) self.CONFIG = CONFIG self.region = region self.lat1, self.lat2 = region[0], region[1] self.lon1, self.lon2 = region[2], region[3] self.db_root_passwd = passwd self.engine = None self.session = None self.spa_resolu = self.CONFIG['grid']['spatial_resolution'] utils.setup_database(self, Base) if run: self.setup_grid()
def __init__(self, CONFIG, period, region, basin, passwd): self.CONFIG = CONFIG self.period = period self.region = region self.db_root_passwd = passwd self.engine = None self.session = None self.basin = basin self.logger = logging.getLogger(__name__) utils.setup_database(self, Base) self.years = [ x for x in range(self.period[0].year, self.period[1].year + 1) ] self.combine()
def __init__(self, CONFIG, period, region, passwd): self.CONFIG = CONFIG self.period = period self.region = region self.db_root_passwd = passwd self.engine = None self.session = None self.logger = logging.getLogger(__name__) self.years = [ x for x in range(self.period[0].year, self.period[1].year + 1) ] utils.setup_database(self, Base) self.download() utils.reset_signal_handler() self.read()
def __init__(self, CONFIG, period, region, passwd, work_mode): self.logger = logging.getLogger(__name__) self.CONFIG = CONFIG self.period = period self.db_root_passwd = passwd self.work_mode = work_mode self.engine = None self.session = None self.years = [ x for x in range(self.period[0].year, self.period[1].year + 1) ] self.lat1, self.lat2 = region[0], region[1] self.lon1, self.lon2 = region[2], region[3] self.zorders = { 'coastlines': 4, 'mapboundary': 0, 'continents': 8, 'station': 9, 'contour': 3, 'contourf': 2, 'wedge': 7, 'grid': 10 } utils.setup_database(self, Base) # self.read_scs_stations() self.grid_lons = None self.grid_lats = None self.grid_x = None self.grid_y = None # Load 4 variables above utils.load_grid_lonlat_xy(self) # self.if_station_on_land() # self.plot_stations_on_map() if self.work_mode != '': self.download_and_read_scs_data()
def __init__(self, CONFIG, period, basin, sources, passwd): self.logger = logging.getLogger(__name__) self.CONFIG = CONFIG self.period = period self.basin = basin self.db_root_passwd = passwd self.engine = None self.session = None self.logger = logging.getLogger(__name__) utils.setup_database(self, Base) self.period_str = [] for dt in self.period: self.period_str.append(dt.strftime('%Y-%m-%d %H:%M:%S')) src_name_priority = { 'smap_prediction': 1, 'smap': 2, 'era5': 3, } # if len(sources) != 2: # self.logger.error('Sources length should be 2') # exit(1) self.sources = sorted(sources, key=src_name_priority.get) self.src_1 = self.sources[0] if len(sources) == 2: self.src_2 = self.sources[1] src_plot_names_mapper = { 'smap_prediction': 'HYBRID', 'smap': 'SMAP', 'era5': 'ERA5', } self.src_plot_names = [ src_plot_names_mapper[src] for src in self.sources ] # self.show_hist_of_matchup() # breakpoint() # if 'smap_prediction' in self.sources: # self.add_dist2coast() self.compare_two_sources()
def __init__(self, CONFIG, period, basin, passwd): self.CONFIG = CONFIG self.period = period self.db_root_passwd = passwd self.engine = None self.session = None self.basin = basin self.logger = logging.getLogger(__name__) utils.setup_database(self, Base) # Get IBTrACS table table_name = self.CONFIG['ibtracs']['table_name'][self.basin] IBTrACS = utils.get_class_by_tablename(self.engine, table_name) self.tc_query = self.session.query(IBTrACS).filter( IBTrACS.date_time >= self.period[0], IBTrACS.date_time <= self.period[1]) self.tc_query_num = self.tc_query.count() self.detect_rapid_intensification()
def __init__(self, CONFIG, period, region, basin, passwd): self.CONFIG = CONFIG self.period = period self.region = region self.db_root_passwd = passwd self.engine = None self.session = None self.basin = basin self.logger = logging.getLogger(__name__) self.years = [ x for x in range(self.period[0].year, self.period[1].year + 1) ] self.lat1, self.lat2 = self.region[0], self.region[1] self.lon1, self.lon2 = self.region[2], self.region[3] self.download(self.basin) utils.reset_signal_handler() utils.setup_database(self, Base) self.read(self.basin, region_restriction=False)
def __init__(self, CONFIG, period, region, passwd, work, save_disk, work_mode, vars_mode): self.CONFIG = CONFIG self.period = period self.region = region self.db_root_passwd = passwd self.engine = None self.session = None self.save_disk = save_disk self.logger = logging.getLogger(__name__) self.lat1, self.lat2 = region[0], region[1] self.lon1, self.lon2 = region[2], region[3] self.years = [ x for x in range(self.period[0].year, self.period[1].year + 1) ] self.main_hours = self.CONFIG['era5']['main_hours'] self.edge = self.CONFIG['regression']['edge_in_degree'] self.cdsapi_client = cdsapi.Client() self.vars = self.CONFIG['era5']['vars'] self.surface_pres_lvl = '1000' self.spa_resolu = self.CONFIG['era5']['spatial_resolution'] self.lat_grid_points = [ y * self.spa_resolu - 90 for y in range(self.CONFIG['era5']['lat_grid_points_number']) ] self.lon_grid_points = [ x * self.spa_resolu for x in range(self.CONFIG['era5']['lon_grid_points_number']) ] # Size of threeD grid points around TC center self.threeD_grid = dict() self.threeD_grid['lat_axis'] = self.threeD_grid['lon_axis'] = int( self.edge / self.spa_resolu) + 1 self.threeD_grid['height'] = len(self.CONFIG['era5']['pres_lvls']) self.twoD_grid = dict() self.twoD_grid['lat_axis'] = self.twoD_grid['lon_axis'] = int( self.edge / self.spa_resolu) + 1 self.wind_radii = self.CONFIG['wind_radii'] self.grid_pts = dict() self.grid_pts['era5'] = dict() self.grid_pts['era5']['lat'] = [ y * self.spa_resolu - 90 for y in range(self.CONFIG['era5']['lat_grid_points_number']) ] self.grid_pts['era5']['lon'] = [ x * self.spa_resolu for x in range(self.CONFIG['era5']['lon_grid_points_number']) ] self.zorders = self.CONFIG['plot']['zorders']['compare'] utils.setup_database(self, Base) self.grid_lons = None self.grid_lats = None self.grid_x = None self.grid_y = None # Load 4 variables above utils.load_grid_lonlat_xy(self) if work: # work_mode: 'scs' / 'tc' # vars_mode: 'surface_all_vars' / 'threeD' / 'surface_wind' self.download_and_read(work_mode, vars_mode)
def __init__(self, CONFIG, train_period, train_test_split_dt, region, basin, passwd, save_disk, instructions, smogn_target, tag): self.logger = logging.getLogger(__name__) self.CONFIG = CONFIG self.train_period = train_period self.train_test_split_dt = train_test_split_dt self.region = region self.db_root_passwd = passwd self.engine = None self.session = None self.save_disk = save_disk self.basin = basin self.instructions = instructions self.smogn_target = smogn_target self.tag = tag self.edge = self.CONFIG['regression']['edge_in_degree'] self.spa_resolu = self.CONFIG['era5']['spatial_resolution'] self.lat_grid_points = [y * self.spa_resolu - 90 for y in range( self.CONFIG['era5']['lat_grid_points_number'])] self.lon_grid_points = [x * self.spa_resolu for x in range( self.CONFIG['era5']['lon_grid_points_number'])] self.model_dir = dict() self.model_dir['lgb'] = self.CONFIG['regression']['dirs'][ 'tc']['lightgbm']['model'] self.validation_split = 0.2 self.predict_table_name_prefix = 'predicted_smap_tc' self.compare_zorders = self.CONFIG['plot']['zorders'][ 'compare'] self.wind_radii = self.CONFIG['wind_radii'] utils.setup_database(self, Base) self.y_name = self.CONFIG['regression']['target']['smap_era5'] # self.y_name = self.CONFIG['regression']['target']['sfmr_era5'] self.set_variables() self.read_smap_era5() self.show_dataset_shape() self.lgb_train = lgb.Dataset(self.X_train, self.y_train, free_raw_data=False) self.lgb_eval = lgb.Dataset(self.X_test, self.y_test, reference=self.lgb_train, free_raw_data=False) if 'vote' in self.instructions: self.voting_regressor() if 'lgb' in self.instructions: if 'optimize' in self.instructions: self.optimize_lgb(maxevals=3) elif 'compare_best' in self.instructions: self.compare_lgb_best_regressor() elif 'best' in self.instructions: self.lgb_best_regressor() elif 'default' in self.instructions: self.lgb_default_regressor() else: self.logger.error('Nothing happen')
def __init__(self, CONFIG, period, region, basin, passwd, save_disk, simulate_instructions): self.CONFIG = CONFIG self.period = period self.region = region self.db_root_passwd = passwd self.save_disk = save_disk self.engine = None self.session = None self.simulate_instructions = list(set(simulate_instructions)) self.basin = basin self.logger = logging.getLogger(__name__) utils.setup_database(self, Base) self.lat1, self.lat2 = region[0], region[1] self.lon1, self.lon2 = region[2], region[3] self.spa_resolu = dict() self.spa_resolu['era5'] = dict() self.spa_resolu['era5']['atm'] = self.CONFIG['era5'][ 'spatial_resolution'] self.spa_resolu['era5']['ocean'] = self.CONFIG['era5'][ 'ocean_spatial_resolution'] self.lats = dict() self.lons = dict() self.lats['era5'] = [ y * self.spa_resolu['era5']['atm'] - 90 for y in range(721) ] self.lons['era5'] = [ x * self.spa_resolu['era5']['atm'] for x in range(1440) ] self.spa_resolu['smap'] = self.CONFIG['rss']['spatial_resolution'] self.lats['smap'] = [ y * self.spa_resolu['smap'] - 89.875 for y in range(720) ] self.lons['smap'] = [ x * self.spa_resolu['smap'] + 0.125 for x in range(1440) ] self.reg_edge = self.CONFIG['regression']['edge_in_degree'] self.half_reg_edge = self.reg_edge / 2 self.half_reg_edge_grid_intervals = int(self.half_reg_edge / self.spa_resolu['era5']['atm']) self.pres_lvls = self.CONFIG['era5']['pres_lvls'] self.grid_lons = None self.grid_lats = None self.grid_x = None self.grid_y = None # Load 4 variables above utils.load_grid_lonlat_xy(self) self.zorders = self.CONFIG['plot']['zorders']['compare'] utils.reset_signal_handler() self.tc_names = [] for name in self.simulate_instructions: self.tc_names.append(name.upper()) self.simulate_smap_windspd()
def __init__(self, CONFIG, period, region, passwd, work_mode): self.CONFIG = CONFIG self.period = period self.region = region self.db_root_passwd = passwd self.engine = None self.session = None self.logger = logging.getLogger(__name__) utils.setup_database(self, Base) self.lat1, self.lat2 = region[0], region[1] self.lon1, self.lon2 = region[2], region[3] self.spa_resolu = dict() self.spa_resolu['ccmp'] = self.CONFIG['ccmp']\ ['spatial_resolution'] self.spa_resolu['grid'] = self.CONFIG['grid']\ ['spatial_resolution'] self.grid_pts = dict() self.grid_pts['ccmp'] = dict() self.grid_pts['ccmp']['lat'] = [ y * self.spa_resolu['ccmp'] - 78.375 for y in range(self.CONFIG['ccmp']['lat_grid_points_number']) ] self.grid_pts['ccmp']['lon'] = [ x * self.spa_resolu['ccmp'] + 0.125 for x in range(self.CONFIG['ccmp']['lon_grid_points_number']) ] self.grid_lons = None self.grid_lats = None self.grid_x = None self.grid_y = None # Load 4 variables above utils.load_grid_lonlat_xy(self) self.zorders = self.CONFIG['plot']['zorders']['scs_basemap'] self._get_region_corners_indices() self.root_url = self.CONFIG['ccmp']['url'] self.filename_prefix = self.CONFIG['ccmp']['filename']\ ['prefix'] self.filename_suffix = self.CONFIG['ccmp']['filename']\ ['suffix'] self.root_dir = self.CONFIG['ccmp']['dir'] utils.set_format_custom_text(self.CONFIG['ccmp']['filename_length']) if work_mode == 'fetch': self.download('tc') self.read() elif work_mode == 'compare': self.compare_ccmp_with_ibtracs() elif work_mode == 'fetch_and_compare': self.download('tc') self.read() self.compare_ccmp_with_ibtracs()