def __init__(
self,
start_date: datetime = None,
rnday: Union[int, float, timedelta] = timedelta(days=5.),
product='medium_range_mem1',
verbose=False,
fallback=True,
):
"""This will download the latest National Water Model data.
NetCDF files are saved to the system's temporary directory.
The AWS data goes back 30 days. For requesting hindcast data from
before we need a different data source
"""
self.start_date = nearest_cycle_date() if start_date is None \
else localize_datetime(start_date).astimezone(pytz.utc)
self.rnday = rnday if isinstance(rnday, timedelta) \
else timedelta(days=rnday)
self.product = product
self.fallback = fallback
# if the model start_date aligns to a "zero" with the NWM data, then
# fetching the data is trivial
self._files = {dt: None for dt in self.timevector}
if self.start_date == nearest_cycle_date(self.start_date):
self._fetch_data()
# if they don't align then we need to inject a "zero" entry at the
# beginning. My suggestion is to repeat the first value. Program will
# raise if that is the case, so we can address this special case later.
# TODO: Put a "zero" entry if start_date and NWM dates do not align.
else:
raise NotImplementedError(
f'Model start_date={str(self.start_date)} is not a "pivot" '
'time.')
def __init__(self, start_date=None, rnday=2, bbox=None):
self.start_date = nearest_cycle_date() if start_date is None else \
localize_datetime(start_date).astimezone(pytz.utc)
self.rnday = rnday if isinstance(rnday, timedelta) else \
timedelta(days=rnday)
if self.start_date != nearest_cycle_date(self.start_date):
raise NotImplementedError(
'Argment start_date is does not align with any HRRR cycle '
'times.')
self._files = {_: None for _ in np.arange(
self.start_date,
self.start_date + self.rnday + self.output_interval,
self.output_interval
).astype(datetime)}
for dt in self.pivot_times:
if None not in list(self._files.values()):
break
base_url = BASE_URL + f'/{self.product}' + \
f'/hrrr{pivot_time(dt).strftime("%Y%m%d")}'
# cycle
for cycle in reversed(range(0, 24, int(self.output_interval.total_seconds() / 3600))):
test_url = f'{base_url}/' + \
f'hrrr_sfc.t{cycle:02d}z'
try:
logger.info(f'Checking url: {test_url}')
nc = Dataset(test_url)
logger.info('Success!')
except OSError as e:
if e.errno == -70:
print()
continue
elif e.errno == -73:
nc = False
def retry():
try:
return Dataset(test_url)
except Exception:
return False
while not isinstance(nc, Dataset):
nc = retry()
else:
raise e
file_dates = self.get_nc_datevector(nc)
for _datetime in reversed(list(self._files.keys())):
if _datetime in file_dates:
if self._files[_datetime] is None:
self._files[_datetime] = nc
if not any(nc is None for nc in self._files.values()):
break
missing_records = [dt for dt, nc in self._files.items() if nc is None]
if len(missing_records) > 0:
raise ValueError(f'No HRRR data for dates: {missing_records}.')
self._bbox = self._modified_bbox(bbox)
def nearest_cycle_date(self) -> datetime:
return nearest_cycle_date(self.start_date)
def fetch_data(
self,
start_date: datetime = None,
rnday: Union[float, timedelta] = 4,
air: bool = True,
prc: bool = True,
rad: bool = True,
bbox: Bbox = None,
):
"""Fetches HRRR data from NOMADS server. """
logger.info('Fetching HRRR data.')
self.start_date = nearest_cycle_date() if start_date is None else \
localize_datetime(start_date).astimezone(pytz.utc)
self.rnday = rnday if isinstance(rnday, timedelta) else \
timedelta(days=rnday)
inventory = HRRRInventory(
self.start_date,
self.rnday + self.output_interval,
bbox
)
nx_grid, ny_grid = inventory.xy_grid()
if air is True:
with Dataset(
self.tmpdir /
f"air_{inventory.product}_"
f"{str(self.start_date)}.nc",
'w', format='NETCDF3_CLASSIC'
) as dst:
# global attributes
dst.setncatts({"Conventions": "CF-1.0"})
# dimensions
dst.createDimension('nx_grid', nx_grid.shape[1])
dst.createDimension('ny_grid', ny_grid.shape[0])
dst.createDimension('time', None)
# variables
# lon
dst.createVariable('lon', 'f4', ('ny_grid', 'nx_grid'))
dst['lon'].long_name = "Longitude"
dst['lon'].standard_name = "longitude"
dst['lon'].units = "degrees_east"
dst['lon'][:] = nx_grid
# lat
dst.createVariable('lat', 'f4', ('ny_grid', 'nx_grid'))
dst['lat'].long_name = "Latitude"
dst['lat'].standard_name = "latitude"
dst['lat'].units = "degrees_north"
dst['lat'][:] = ny_grid
# time
dst.createVariable('time', 'f4', ('time',))
dst['time'].long_name = 'Time'
dst['time'].standard_name = 'time'
date = pivot_time(self.start_date)
dst['time'].units = f'days since {date.year}-{date.month}'\
f'-{date.day} 00:00'\
f'{date.tzinfo}'
dst['time'].base_date = (date.year, date.month, date.day, 0)
dst['time'][:] = inventory.get_sflux_timevector()
for var in AirComponent.var_types:
dst.createVariable(
var,
'f4',
('time', 'ny_grid', 'nx_grid')
)
logger.info(f'Put field {var}')
inventory.put_sflux_field(getattr(self, f'{var}_name'), dst, var)
# prmsl
dst['prmsl'].long_name = "Pressure reduced to MSL"
dst['prmsl'].standard_name = "air_pressure_at_sea_level"
dst['prmsl'].units = "Pa"
# spfh
dst['spfh'].long_name = "Surface Specific Humidity "\
"(2m AGL)"
dst['spfh'].standard_name = "specific_humidity"
dst['spfh'].units = "1"
# stmp
dst['stmp'].long_name = "Surface Air Temperature (2m AGL)"
dst['stmp'].standard_name = "air_temperature"
dst['stmp'].units = "K"
# uwind
dst['uwind'].long_name = "Surface Eastward Air Velocity "\
"(10m AGL)"
dst['uwind'].standard_name = "eastward_wind"
dst['uwind'].units = "m/s"
# vwind
dst['vwind'].long_name = "Surface Northward Air Velocity "\
"(10m AGL)"
dst['vwind'].standard_name = "northward_wind"
dst['vwind'].units = "m/s"
if prc is True:
with Dataset(
self.tmpdir /
f"prc_{inventory.product}_"
f"{str(self.start_date)}.nc",
'w', format='NETCDF3_CLASSIC'
) as dst:
# global attributes
dst.setncatts({"Conventions": "CF-1.0"})
# dimensions
dst.createDimension('nx_grid', nx_grid.shape[1])
dst.createDimension('ny_grid', ny_grid.shape[0])
dst.createDimension('time', None)
# lon
dst.createVariable('lon', 'f4', ('ny_grid', 'nx_grid'))
dst['lon'].long_name = "Longitude"
dst['lon'].standard_name = "longitude"
dst['lon'].units = "degrees_east"
dst['lon'][:] = nx_grid
# lat
dst.createVariable('lat', 'f4', ('ny_grid', 'nx_grid'))
dst['lat'].long_name = "Latitude"
dst['lat'].standard_name = "latitude"
dst['lat'].units = "degrees_north"
dst['lat'][:] = ny_grid
# time
dst.createVariable('time', 'f4', ('time',))
dst['time'].long_name = 'Time'
dst['time'].standard_name = 'time'
date = pivot_time(self.start_date)
dst['time'].units = f'days since {date.year}-{date.month}'\
f'-{date.day} 00:00'\
f'{date.tzinfo}'
dst['time'].base_date = (date.year, date.month, date.day, 0)
dst['time'][:] = inventory.get_sflux_timevector()
for var in PrcComponent.var_types:
dst.createVariable(var, float,
('time', 'ny_grid', 'nx_grid'))
logger.info(f'Put field {var}')
inventory.put_sflux_field(getattr(self, f'{var}_name'), dst, var)
# prate
dst['prate'].long_name = "Surface Precipitation Rate"
dst['prate'].standard_name = "air_pressure_at_sea_level"
dst['prate'].units = "kg/m^2/s"
if rad is True:
with Dataset(
self.tmpdir /
f"rad_{inventory.product}_"
f"{str(self.start_date)}.nc",
'w', format='NETCDF3_CLASSIC'
) as dst:
# global attributes
dst.setncatts({"Conventions": "CF-1.0"})
# dimensions
dst.createDimension('nx_grid', nx_grid.shape[1])
dst.createDimension('ny_grid', ny_grid.shape[0])
dst.createDimension('time', None)
# lon
dst.createVariable('lon', 'f4', ('ny_grid', 'nx_grid'))
dst['lon'].long_name = "Longitude"
dst['lon'].standard_name = "longitude"
dst['lon'].units = "degrees_east"
dst['lon'][:] = nx_grid
# lat
dst.createVariable('lat', 'f4', ('ny_grid', 'nx_grid'))
dst['lat'].long_name = "Latitude"
dst['lat'].standard_name = "latitude"
dst['lat'].units = "degrees_north"
dst['lat'][:] = ny_grid
# time
dst.createVariable('time', 'f4', ('time',))
dst['time'].long_name = 'Time'
dst['time'].standard_name = 'time'
date = pivot_time(self.start_date)
dst['time'].units = f'days since {date.year}-{date.month}'\
f'-{date.day} 00:00'\
f'{date.tzinfo}'
dst['time'].base_date = (date.year, date.month, date.day, 0)
dst['time'][:] = inventory.get_sflux_timevector()
for var in RadComponent.var_types:
dst.createVariable(var, float,
('time', 'ny_grid', 'nx_grid'))
logger.info(f'Put field {var}')
inventory.put_sflux_field(getattr(self, f'{var}_name'), dst, var)
# dlwrf
dst['dlwrf'].long_name = "Downward Long Wave Radiation "\
"Flux"
dst['dlwrf'].standard_name = "surface_downwelling_"\
"longwave_flux_in_air"
dst['dlwrf'].units = "W/m^2"
# dswrf
dst['dswrf'].long_name = "Downward Short Wave Radiation "\
"Flux"
dst['dswrf'].standard_name = "surface_downwelling_"\
"shortwave_flux_in_air"
dst['dswrf'].units = "W/m^2"
self.resource = self.tmpdir
self.air = AirComponent(self.fields)
self.prc = PrcComponent(self.fields)
self.rad = RadComponent(self.fields)