def test_rp_calculated_for_all_necessary_vars(self):
yearmon = dates.get_next_yearmon(self.config.historical_yearmons()[-1])
target = dates.add_months(yearmon, 3)
model = self.config.models()[0]
member = self.config.forecast_ensemble_members(model, yearmon)[0]
steps = self.get_steps(yearmon, yearmon)
ws = self.config.workspace()
# 1-month observed rp
rp_step = step_for_target(steps,
ws.return_period(yearmon=yearmon, window=1))
for v in self.config.lsm_rp_vars() + self.config.forcing_rp_vars():
self.assertIn(
ws.fit_obs(var=v,
window=1,
month=dates.parse_yearmon(yearmon)[1]),
rp_step.dependencies)
# 1-month forecast rp
rp_step = step_for_target(
steps,
ws.return_period(yearmon=yearmon,
model=model,
target=target,
member=member,
window=1))
for v in self.config.lsm_rp_vars() + self.config.forcing_rp_vars():
self.assertIn(
ws.fit_obs(var=v,
window=1,
month=dates.parse_yearmon(target)[1]),
rp_step.dependencies)
# 3-month observed rp
rp_step = step_for_target(steps,
ws.return_period(yearmon=yearmon, window=3))
for v, stats in self.config.lsm_integrated_vars().items():
for stat in stats:
self.assertIn(
ws.fit_obs(var=v,
window=3,
stat=stat,
month=dates.parse_yearmon(yearmon)[1]),
rp_step.dependencies)
# 3-month forecast rp
rp_step = step_for_target(
steps,
ws.return_period(yearmon=yearmon,
model=model,
target=target,
member=member,
window=3))
for v in self.config.lsm_integrated_var_names():
self.assertTrue(
ws.fit_obs(
var=v, window=3, month=dates.parse_yearmon(target)[1]) in
rp_step.dependencies)
def available_hindcasts(self, target_month: int, lead: int) -> List[str]:
forecast_month = target_month - lead
if forecast_month < 1:
forecast_month += 12
for forecast_year in range(self.min_fit_year - 1,
self.max_fit_year + 1):
target = dates.add_months(
dates.format_yearmon(forecast_year, forecast_month), lead)
target_year, _ = dates.parse_yearmon(target)
if target_year > self.max_fit_year:
continue
assert dates.parse_yearmon(target)[1] == target_month
for day in HINDCAST_DATES_FOR_MONTH[forecast_month]:
for hour in (0, 6, 12, 18):
timestamp = '{:04d}{:02d}{:02d}{:02d}'.format(
forecast_year, forecast_month, day, hour)
if timestamp in MISSING_HINDCASTS and target in MISSING_HINDCASTS[
timestamp]:
continue
if timestamp in CORRUPT_HINDCASTS and target in CORRUPT_HINDCASTS[
timestamp]:
continue
yield timestamp, target
def p_wetdays(self, *, yearmon: str, target: str, member: str) -> Vardef:
_, month = dates.parse_yearmon(target)
return paths.Vardef(
os.path.join(self.subdir(),
'mean_p_wetdays_month_{:02d}.nc'.format(month)),
'pWetDays')
def precip_monthly(self, *, yearmon: str, target: str,
member: str) -> Vardef:
_, month = dates.parse_yearmon(target)
return paths.Vardef(
os.path.join(self.subdir(),
'mean_prate_month_{:02d}.nc'.format(month)), 'Pr')
def prep_steps(self, *, yearmon: str) -> List[Step]:
"""
Prep steps are data preparation tasks that are executed once per model iteration.
They may include downloading, unpackaging, aggregation, or conversion of data inputs.
:param yearmon: yearmon of model iteration
:return: a list of Steps
"""
steps = []
year, month = dates.parse_yearmon(yearmon)
# Extract netCDF of monthly temperature from full binary file
steps += ghcn_cams.extract_monthly_temperature(grib_file=self.ghcn_cams_grib(),
output_filename=self.temp_monthly(yearmon=yearmon).file,
yearmon=yearmon)
steps += precl.download_precl(yearmon=yearmon,
output_filename=self.precip_monthly(yearmon=yearmon).file)
if year >= 1979:
steps += cpc_daily_precipitation.download_monthly_precipitation(
yearmon=yearmon,
workdir=os.path.join(self.source,
'NCEP',
'daily_precip'),
wetdays_fname=self.p_wetdays(yearmon=yearmon).file)
return steps
def prep_steps(self, *, yearmon: str) -> List[Step]:
"""
Prep steps are data preparation tasks that are executed once per model iteration.
They may include downloading, unpackaging, aggregation, or conversion of data inputs.
:param yearmon: yearmon of model iteration
:return: a list of Steps
"""
steps = []
year, month = dates.parse_yearmon(yearmon)
# Extract netCDF of monthly precipitation from full binary file
steps.append(
Step(
targets=self.precip_monthly(yearmon=yearmon).file,
dependencies=self.full_precip_file(),
commands=[[
os.path.join('{BINDIR}', 'utils',
'noaa_global_leaky_bucket',
'read_binary_grid.R'),
'--input',
self.full_precip_file(),
'--update_url',
'ftp://ftp.cpc.ncep.noaa.gov/wd51yf/global_monthly/gridded_binary/p.long',
'--output',
self.precip_monthly(yearmon=yearmon).file,
'--var',
'P',
'--yearmon',
yearmon,
]]))
# Extract netCDF of monthly temperature from full binary file
steps.append(
Step(
targets=self.temp_monthly(yearmon=yearmon).file,
dependencies=self.full_temp_file(),
commands=[[
os.path.join('{BINDIR}', 'utils',
'noaa_global_leaky_bucket',
'read_binary_grid.R'), '--input',
self.full_temp_file(), '--update_url',
'ftp://ftp.cpc.ncep.noaa.gov/wd51yf/global_monthly/gridded_binary/t.long',
'--output',
self.temp_monthly(yearmon=yearmon).file, '--var', 'T',
'--yearmon', yearmon
]]))
if year >= 1979:
steps += cpc_daily_precipitation.download_monthly_precipitation(
yearmon=yearmon,
workdir=os.path.join(self.source, 'NCEP', 'daily_precip'),
wetdays_fname=self.p_wetdays(yearmon=yearmon).file)
return steps
def p_wetdays(self, *, yearmon: str) -> paths.Vardef:
year, month = dates.parse_yearmon(yearmon)
if year < 1979:
return self.mean_p_wetdays(month)
else:
return paths.Vardef(
os.path.join(self.source, 'NCEP', 'wetdays',
'wetdays_{yearmon}.nc'.format(yearmon=yearmon)),
'pWetDays')
def prep_steps(self, *, yearmon: str, target: str,
member: str) -> List[Step]:
steps = []
_, nmme_month = dates.parse_yearmon(wsim_to_nmme_yearmon(yearmon))
# Hack to only download these once although they are required for
# all members / forecast targets
if int(member) == 1 and target == dates.add_months(yearmon, 1):
steps += self.download_realtime_anomalies(
nmme_yearmon=wsim_to_nmme_yearmon(yearmon))
output = self.forecast_raw(yearmon=yearmon,
target=target,
member=member).split('::')[0]
steps.append(
Step(targets=output,
dependencies=[
self.forecast_anom(
nmme_yearmon=wsim_to_nmme_yearmon(yearmon),
varname='T'),
self.forecast_anom(
nmme_yearmon=wsim_to_nmme_yearmon(yearmon),
varname='Pr'),
self.forecast_clim(nmme_month=nmme_month, varname='T'),
self.forecast_clim(nmme_month=nmme_month, varname='Pr')
],
commands=[[
os.path.join('{BINDIR}', 'utils', 'nmme',
'extract_nmme_forecast.R'), '--clim_precip',
self.forecast_clim(nmme_month=nmme_month,
varname='Pr'), '--clim_temp',
self.forecast_clim(nmme_month=nmme_month, varname='T'),
'--anom_precip',
self.forecast_anom(
nmme_yearmon=wsim_to_nmme_yearmon(yearmon),
varname='Pr'), '--anom_temp',
self.forecast_anom(
nmme_yearmon=wsim_to_nmme_yearmon(yearmon),
varname='T'), '--member', member, '--lead',
str(
dates.get_lead_months(wsim_to_nmme_yearmon(yearmon),
target)), '--output', output
]]))
return steps
def last_7_days_of_previous_month(yearmon: str,
lag_hours: Optional[int] = None
) -> List[str]:
# Build an ensemble of 28 forecasts by taking the four
# forecasts issued on each of the last 7 days of the month.
last_day = dates.get_last_day_of_month(yearmon)
year, month = dates.parse_yearmon(yearmon)
members = [
datetime.datetime(year, month, day, hour)
for day in range(last_day - 6, last_day + 1)
for hour in (0, 6, 12, 18)
]
if lag_hours is not None:
members = [
m for m in members if datetime.datetime.utcnow() -
m > datetime.timedelta(hours=lag_hours)
]
return [m.strftime('%Y%m%d%H') for m in members]
def p_wetdays(self, *, yearmon):
year, mon = parse_yearmon(yearmon)
if year >= 1979:
return Vardef('wetdays_{}.tif'.format(yearmon), '1')
else:
return Vardef('wetdays_norms_{}.tif'.format(mon), '1')
def p_wetdays(self, *, yearmon=None, target, member=None):
_, month = dates.parse_yearmon(target)
return self.observed().mean_p_wetdays(month=month)
def precip_monthly(self, *, yearmon: str) -> paths.Vardef:
year, _ = dates.parse_yearmon(yearmon)
return paths.Vardef(os.path.join(self.source, 'PRECL', str(year), 'precl_{yearmon}.nc'.format(yearmon=yearmon)),
'Pr')
def temp_monthly(self, *, yearmon: str) -> paths.Vardef:
year, _ = dates.parse_yearmon(yearmon)
return paths.Vardef(os.path.join(self.source, 'GHCN_CAMS', str(year), 'ghcn_cams_{yearmon}.nc'.format(yearmon=yearmon)), 'T')