def __init__(self, CONFIG, period, region, passwd):
self.SFMR_CONFIG = CONFIG['sfmr']
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.logger = logging.getLogger(__name__)
utils.setup_database(self, Base)
self.years = [
x for x in range(self.period[0].year, self.period[1].year + 1)
]
brief_info = self.get_all_hurricanes_brief_info()
self.download_and_update_by_brief_info(brief_info)
return
self._gen_all_year_hurr()
self._extract_year_hurr()
self.download()
utils.reset_signal_handler()
read_all = False
self._extract_year_hurr_file_path(read_all)
def download_and_update_by_brief_info(self, brief_info):
# pickle_dir = self.SFMR_CONFIG['dirs']['brief_info']
# pickle_path = f'{pickle_dir}brief_info.pkl'
# with open(pickle_path, 'rb') as f:
# brief_info = pickle.load(f)
# Download all SFMR files during period
utils.setup_signal_handler()
utils.set_format_custom_text(self.SFMR_CONFIG['data_name_length'])
self.download_with_brief_info(brief_info)
# Read all of them to update brief information
utils.reset_signal_handler()
brief_info = self.update_brief_info(brief_info)
brief_info_list = []
for year in brief_info:
for info in brief_info[year]:
if info is not None:
brief_info_list.append(info)
else:
brief_info[year].remove(info)
utils.bulk_insert_avoid_duplicate_unique(
brief_info_list,
self.CONFIG['database']['batch_size']['insert'],
SFMRDetail, ['filename'],
self.session,
check_self=True)
self.brief_info = brief_info
return True
def __init__(self, CONFIG, period, region, passwd):
self.STDMET_CONFIG = CONFIG['stdmet']
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.years = [x for x in range(self.period[0].year,
self.period[1].year+1)]
self.stations = set()
self.logger = logging.getLogger(__name__)
self._analysis_and_save_relation()
self._extract_relation()
# self.download()
# self.download_all_stations_no_limit()
utils.reset_signal_handler()
utils.setup_database(self, Base)
# self.read()
self.show()
def __init__(self, CONFIG, period, region, passwd):
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.logger = logging.getLogger(__name__)
self.years = [
x for x in range(self.period[0].year, self.period[1].year + 1)
]
utils.setup_database(self, Base)
self.download()
utils.reset_signal_handler()
self.read()
def __init__(self, CONFIG, period, region, basin, passwd):
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.engine = None
self.session = None
self.basin = basin
self.logger = logging.getLogger(__name__)
self.years = [
x for x in range(self.period[0].year, self.period[1].year + 1)
]
self.lat1, self.lat2 = self.region[0], self.region[1]
self.lon1, self.lon2 = self.region[2], self.region[3]
self.download(self.basin)
utils.reset_signal_handler()
utils.setup_database(self, Base)
self.read(self.basin, region_restriction=False)
def download_ccmp_on_one_day(self, dt_cursor):
utils.setup_signal_handler()
filename = (f"""{self.filename_prefix}"""
f"""{dt_cursor.strftime('%Y%m%d')}"""
f"""{self.filename_suffix}""")
url_prefix = (f"""{self.root_url}/Y{dt_cursor.year}/"""
f"""M{str(dt_cursor.month).zfill(2)}/""")
file_dir = (f"""{self.root_dir}Y{dt_cursor.year}/"""
f"""M{str(dt_cursor.month).zfill(2)}/""")
file_path = f'{file_dir}{filename}'
file_url = f'{url_prefix}{filename}'
if os.path.exists(file_path):
return file_path
self.logger.info((f"""Downloading {filename}"""))
os.makedirs(file_dir, exist_ok=True)
utils.download(file_url, file_path, progress=True)
utils.reset_signal_handler()
return file_path
def read(self):
utils.reset_signal_handler()
self.logger.info((f"""Reading CCMP files"""))
# Traverse file path
for file_path in self.files_path:
date_str = file_path.split('_')[3]
vars = Dataset(file_path).variables
date_ = datetime.datetime.strptime(date_str, '%Y%m%d').date()
CCMP = self.create_scs_ccmp_table(date_)
info = f"""Reading {file_path.split('/')[-1]}"""
# Traverse 4 time in one day
for hour_idx, hour in enumerate(range(0, 24, 6)):
print(f"""\r{info} on {str(hour).zfill(2)}:00""", end='')
one_hour_scs_ccmp = []
time = datetime.time(hour, 0, 0)
dt = datetime.datetime.combine(date_, time)
subset = dict()
var_names = ['nobs', 'uwnd', 'vwnd']
for var_name in var_names:
subset[var_name] = vars[var_name][hour_idx][
self.lat1_index:self.lat2_index + 1,
self.lon1_index:self.lon2_index + 1]
one_hour_scs_ccmp = self.get_ccmp_of_one_hour(
dt, CCMP, subset, var_names)
# Insert into table
utils.bulk_insert_avoid_duplicate_unique(
one_hour_scs_ccmp, self.CONFIG['database']\
['batch_size']['insert'],
CCMP, ['datetime_x_y'], self.session,
check_self=True)
utils.delete_last_lines()
print(f"""{info}: Done""")
def __init__(self, CONFIG, period, region, basin, passwd, save_disk,
simulate_instructions):
self.CONFIG = CONFIG
self.period = period
self.region = region
self.db_root_passwd = passwd
self.save_disk = save_disk
self.engine = None
self.session = None
self.simulate_instructions = list(set(simulate_instructions))
self.basin = basin
self.logger = logging.getLogger(__name__)
utils.setup_database(self, Base)
self.lat1, self.lat2 = region[0], region[1]
self.lon1, self.lon2 = region[2], region[3]
self.spa_resolu = dict()
self.spa_resolu['era5'] = dict()
self.spa_resolu['era5']['atm'] = self.CONFIG['era5'][
'spatial_resolution']
self.spa_resolu['era5']['ocean'] = self.CONFIG['era5'][
'ocean_spatial_resolution']
self.lats = dict()
self.lons = dict()
self.lats['era5'] = [
y * self.spa_resolu['era5']['atm'] - 90 for y in range(721)
]
self.lons['era5'] = [
x * self.spa_resolu['era5']['atm'] for x in range(1440)
]
self.spa_resolu['smap'] = self.CONFIG['rss']['spatial_resolution']
self.lats['smap'] = [
y * self.spa_resolu['smap'] - 89.875 for y in range(720)
]
self.lons['smap'] = [
x * self.spa_resolu['smap'] + 0.125 for x in range(1440)
]
self.reg_edge = self.CONFIG['regression']['edge_in_degree']
self.half_reg_edge = self.reg_edge / 2
self.half_reg_edge_grid_intervals = int(self.half_reg_edge /
self.spa_resolu['era5']['atm'])
self.pres_lvls = self.CONFIG['era5']['pres_lvls']
self.grid_lons = None
self.grid_lats = None
self.grid_x = None
self.grid_y = None
# Load 4 variables above
utils.load_grid_lonlat_xy(self)
self.zorders = self.CONFIG['plot']['zorders']['compare']
utils.reset_signal_handler()
self.tc_names = []
for name in self.simulate_instructions:
self.tc_names.append(name.upper())
self.simulate_smap_windspd()