def extract_between_two_tc_records(self, tc, next_tc):
"""
Notes
-----
In this project, the range of longitude is from 0 to 360, so there
may be data leap near the prime merdian. And the interpolation
should be ajusted.
"""
# Temporal shift
delta = next_tc.date_time - tc.date_time
# Skip interpolating between two TC recors if two neighbouring
# records of TC are far away in time
if delta.days:
self.extract_detail(tc)
return
hours = int(delta.seconds / 3600)
# Skip interpolating between two TC recors if two neighbouring
# records of TC are too close in time
if not hours:
self.extract_detail(tc)
return
Match = utils.create_match_table(self, ['smap', 'era5'])
hit_dt = []
match_dt = []
for h in range(hours):
interped_tc = utils.interp_tc(self, h, tc, next_tc)
same_sid_dt_query = self.session.query(Match).filter(
Match.date_time == interped_tc.date_time,
Match.tc_sid == interped_tc.sid)
same_sid_dt_count = same_sid_dt_query.count()
if not same_sid_dt_count:
continue
elif same_sid_dt_count == 1:
hit_dt.append(interped_tc.date_time)
if same_sid_dt_query[0].match:
match_dt.append(interped_tc.date_time)
else:
self.logger.error((f"""Strange: two or more """
f"""comparison has same sid """
f"""and datetime"""))
breakpoint()
exit()
hit_count = len(hit_dt)
match_count = len(match_dt)
if hit_count == hours and not match_count:
print((f"""[Skip] All internal hours of TC """
f"""{tc.name} between {tc.date_time} """
f"""and {next_tc.date_time}"""))
return
# Extract from the interval between two TC records
if hit_count == hours:
self.extract_with_all_hours_hit(tc, next_tc, hours, match_dt)
else:
self.extract_with_not_all_hours_hit(tc, next_tc, hours)
def extract_with_not_all_hours_hit(self, tc, next_tc, hours,
spatial_temporal_info,
hour_info_pt_idx):
Match = utils.create_match_table(self, ['sfmr', 'era5'])
for h in range(hours):
interped_tc = utils.interp_tc(self, h, tc, next_tc)
SFMRERA5 = self.create_sfmr_era5_table(interped_tc.date_time)
if interped_tc.date_time not in hour_info_pt_idx.keys():
# update corrseponding match
utils.update_one_row_of_match(self, Match, interped_tc, False)
print((f"""[Not exist] SFMR of TC {tc.name} near """
f"""{interped_tc.date_time}"""))
continue
try:
sfmr_success, data, hourtimes, area = \
self.extract_sfmr_around_interped_tc(
spatial_temporal_info,
hour_info_pt_idx[interped_tc.date_time],
interped_tc)
except Exception as msg:
breakpoint()
exit(msg)
if not sfmr_success or not len(data) or not len(hourtimes):
# Normal fail, need continue comparing
utils.update_one_row_of_match(self, Match, interped_tc, False)
print((f"""[Not found] SFMR """
f"""around TC {interped_tc.name} """
f"""on {interped_tc.date_time}"""))
continue
try:
data = utils.add_era5(self, 'sfmr', interped_tc, data,
hourtimes, area)
except Exception as msg:
breakpoint()
exit(msg)
if not len(data):
utils.update_one_row_of_match(self, Match, interped_tc, False)
print((f"""[No matchup] SFMR and ERA5"""
f"""around TC {interped_tc.name} """
f"""on {interped_tc.date_time}"""))
continue
utils.update_one_row_of_match(self, Match, interped_tc, True)
print((f"""[Match] SFMR and ERA5 """
f"""around TC {interped_tc.name} """
f"""on {interped_tc.date_time}"""))
utils.bulk_insert_avoid_duplicate_unique(
data,
self.CONFIG['database']['batch_size']['insert'],
SFMRERA5, ['sfmr_datetime_lon_lat'],
self.session,
check_self=True)
def extract_with_all_hours_hit(self, tc, next_tc, hours, match_dt):
for h in range(hours):
interped_tc = utils.interp_tc(self, h, tc, next_tc)
if interped_tc.date_time not in match_dt:
print((f"""[Skip] matching SMAP and ERA5 """
f"""around TC {interped_tc.name} """
f"""on {interped_tc.date_time}"""))
continue
success = self.extract_detail(interped_tc)
self.info_after_extracting_detail(interped_tc, success, False)
def extract_with_all_hours_hit(self, tc, next_tc, hours, match_dt,
spatial_temporal_info, hour_info_pt_idx):
#
for h in range(hours):
interped_tc = utils.interp_tc(self, h, tc, next_tc)
SFMRERA5 = self.create_sfmr_era5_table(interped_tc.date_time)
if interped_tc.date_time not in match_dt:
print((f"""[Skip] matching SFMR and ERA5 """
f"""around TC {interped_tc.name} """
f"""on {interped_tc.date_time}"""))
continue
sfmr_success, data, hourtimes, area = \
self.extract_sfmr_around_interped_tc(
spatial_temporal_info,
hour_info_pt_idx[interped_tc.date_time],
interped_tc)
if not sfmr_success or not len(data) or not len(hourtimes):
self.logger.error('Match True but fail extracting SFMR')
continue
try:
data = utils.add_era5(self, 'sfmr', interped_tc, data,
hourtimes, area)
except Exception as msg:
breakpoint()
exit(msg)
if not len(data):
self.logger.error('Match True but fail adding ERA5')
continue
print((f"""[Redo] extract SFMR and ERA5"""
f"""around TC {interped_tc.name} """
f"""on {interped_tc.date_time}"""))
utils.bulk_insert_avoid_duplicate_unique(
data,
self.CONFIG['database']['batch_size']['insert'],
SFMRERA5, ['sfmr_datetime_lon_lat'],
self.session,
check_self=True)
def simulate_between_two_tcs(self, tc, next_tc):
# Temporal shift
delta = next_tc.date_time - tc.date_time
# Skip interpolating between two TC recors if two neighbouring
# records of TC are far away in time
if delta.days:
return False
hours = int(delta.seconds / 3600)
# Time interval between two TCs are less than one hour
if not hours:
return False
# `tight_hours` is for the case that period totally falls in
# the interval of two TC records and is shorted than the
# interval.
if (self.period[0] > tc.date_time
or self.period[1] < next_tc.date_time):
tight_hours = int((self.period[1] - self.period[0]).seconds / 3600)
else:
tight_hours = hours
skip_hour_indices = []
for i, h in enumerate(range(hours)):
interped_tc = utils.interp_tc(self, h, tc, next_tc)
if (interped_tc.date_time < self.period[0]
or interped_tc.date_time > self.period[1]):
skip_hour_indices.append(i)
subplots_row, subplots_col, fig_size = \
utils.get_subplots_row_col_and_fig_size(tight_hours)
fig, axes = plt.subplots(subplots_row, subplots_col, figsize=fig_size)
hours_max_windspd = -1
ax_idx = 0
for i, h in enumerate(range(hours)):
if i in skip_hour_indices:
continue
try:
interped_tc = utils.interp_tc(self, h, tc, next_tc)
if isinstance(axes, np.ndarray):
ax = axes.flat[ax_idx]
else:
ax = axes
except Exception as msg:
breakpoint()
exit(msg)
success, hourly_max_windspd = self.simulate_hourly(
interped_tc, fig, ax)
if success:
hours_max_windspd = max(hours_max_windspd, hourly_max_windspd)
ax_idx += 1
if hourly_max_windspd == -1:
return
ax_idx = 0
for i, h in enumerate(range(hours)):
if i in skip_hour_indices:
continue
try:
interped_tc = utils.interp_tc(self, h, tc, next_tc)
if isinstance(axes, np.ndarray):
ax = axes.flat[ax_idx]
else:
ax = axes
except Exception as msg:
breakpoint()
exit(msg)
success, hourly_max_windspd = self.simulate_hourly(
interped_tc, fig, ax, hours_max_windspd, ax_idx)
if success:
ax_idx += 1
print((f"""Simulating SMAP windspd """
f"""of TC {interped_tc.name} on """
f"""{interped_tc.date_time}"""))
else:
print((f"""Skiping simulating SMAP windspd """
f"""of TC {interped_tc.name} """
f"""on {interped_tc.date_time}"""))
fig.tight_layout(pad=0.1)
dt_str = (f"""{tc.date_time.strftime('%Y_%m%d_%H%M')}"""
f"""_"""
f"""{next_tc.date_time.strftime('_%H%M')}""")
fig_dir = self.CONFIG['result']['dirs']['fig']['simulation']
os.makedirs(fig_dir, exist_ok=True)
fig_name = f'{dt_str}_{tc.name}.png'
plt.savefig(f'{fig_dir}{fig_name}', dpi=600)
plt.clf()
def extract_with_not_all_hours_hit(self, tc, next_tc, hours):
for h in range(hours):
interped_tc = utils.interp_tc(self, h, tc, next_tc)
success = self.extract_detail(interped_tc)
self.info_after_extracting_detail(interped_tc, success, True)
def extract_between_two_tc_records(self, tc, next_tc):
"""
Notes
-----
In this project, the range of longitude is from 0 to 360, so
there may be data leap near the prime merdian. And the
interpolation should be ajusted.
"""
# Temporal shift
delta = next_tc.date_time - tc.date_time
# Skip interpolating between two TC recors if two neighbouring
# records of TC are far away in time
if delta.days:
return
hours = int(delta.seconds / 3600)
# Skip interpolating between two TC recors if two neighbouring
# records of TC are too close in time
if not hours:
return
Match = utils.create_match_table(self, ['sfmr', 'era5'])
hit_dt = []
match_dt = []
for h in range(hours):
interped_tc = utils.interp_tc(self, h, tc, next_tc)
same_sid_dt_query = self.session.query(Match).filter(
Match.date_time == interped_tc.date_time,
Match.tc_sid == interped_tc.sid)
same_sid_dt_count = same_sid_dt_query.count()
if not same_sid_dt_count:
continue
elif same_sid_dt_count == 1:
hit_dt.append(interped_tc.date_time)
if same_sid_dt_query[0].match:
match_dt.append(interped_tc.date_time)
else:
self.logger.error((f"""Strange: two or more """
f"""comparison has same sid """
f"""and datetime"""))
breakpoint()
exit()
hit_count = len(hit_dt)
match_count = len(match_dt)
if hit_count == hours and not match_count:
print((f"""[Skip] All internal hours of TC """
f"""{tc.name} between {tc.date_time} """
f"""and {next_tc.date_time}"""))
return
# Check existence of SFMR between two IBTrACS records
existence, spatial_temporal_info = utils.sfmr_exists(self, tc, next_tc)
if not existence:
# First executed here between particular two TCs
if hit_count < hours:
# update match of data sources
utils.update_no_match_between_tcs(self, Match, hours, tc,
next_tc)
print((f"""[Not exist] SFMR of TC {tc.name} between """
f"""{tc.date_time} and {next_tc.date_time}"""))
return
# Round SMFR record to different hours
# hour_info_pt_idx:
# {
# hour_datetime_1: {
# info_idx_1: [pt_idx_1, pt_idx_2, ...],
# info_idx_2: [pt_idx_1, pt_idx_2, ...],
# ...
# }
# hour_datetime_2: {
# ...
# }
# ...
# }
hour_info_pt_idx = utils.sfmr_rounded_hours(self, tc, next_tc,
spatial_temporal_info)
if not len(hour_info_pt_idx):
# First executed here between particular two TCs
if hit_count < hours:
# update match of data sources
utils.update_no_match_between_tcs(self, Match, hours, tc,
next_tc)
print((f"""[Fail rounding to hour] SFMR of TC {tc.name} """
f"""between {tc.date_time} and """
f"""{next_tc.date_time}"""))
return
if hit_count == hours:
self.extract_with_all_hours_hit(tc, next_tc, hours, match_dt,
spatial_temporal_info,
hour_info_pt_idx)
else:
self.extract_with_not_all_hours_hit(tc, next_tc, hours,
spatial_temporal_info,
hour_info_pt_idx)