def make_partial_date_time(date_string):
"""
Convert the fields in `date_string` into a PartialDateTime object. Formats
that are known about are:
YYYMM
YYYYMMDD
:param str date_string: The date string to process
:returns: An Iris PartialDateTime object containing as much information as
could be deduced from date_string
:rtype: iris.time.PartialDateTime
:raises ValueError: If the string is not in a known format.
"""
if len(date_string) == 6:
pdt_str = PartialDateTime(year=int(date_string[0:4]),
month=int(date_string[4:6]))
elif len(date_string) == 8:
pdt_str = PartialDateTime(year=int(date_string[0:4]),
month=int(date_string[4:6]),
day=int(date_string[6:8]))
else:
raise ValueError('Unknown date string format')
return pdt_str
def ref_clim(self, cube, start_y,end_y):
t1 = PartialDateTime(year=start_y)
t2 = PartialDateTime(year=end_y)
constr_1 = iris.Constraint(time=lambda t: t1 <= t.point <= t2)
cube_out = cube.extract(constr_1)
cube_out = cube_out.collapsed('time', iris.analysis.MEAN)
return cube_out
def datetime_constraint(
time_in: datetime, time_max: Optional[datetime] = None
) -> Constraint:
"""
Constructs an iris equivalence constraint from a python datetime object.
Args:
time_in:
The time to be used to build an iris constraint.
time_max:
Optional max time, which if provided leads to a range constraint
being returned up to < time_max.
Returns:
An iris constraint to be used in extracting data at the given time
from a cube.
"""
time_start = PartialDateTime(time_in.year, time_in.month, time_in.day, time_in.hour)
if time_max is None:
time_extract = Constraint(time=lambda cell: cell.point == time_start)
else:
time_limit = PartialDateTime(
time_max.year, time_max.month, time_max.day, time_max.hour
)
time_extract = Constraint(time=lambda cell: time_start <= cell < time_limit)
return time_extract
def lagged_correlation(cube, lag):
end, start = 2010 - lag, 1950 + lag
tnolag = PartialDateTime(year=end)
tlag = PartialDateTime(year=start)
# constrsain a lag and an unlagged cube.
nolag = cube.extract(iris.Constraint(time=lambda t: t.point <= tnolag))
lag = cube.extract(iris.Constraint(time=lambda t: tlag <= t.point))
# define coords to unify the dimensions where the data lays.
time = DimCoord(nolag.coord('time').points, standard_name='time')
latitude = DimCoord(cube.coord('latitude').points,
standard_name='latitude',
units='degrees')
longitude = DimCoord(cube.coord('longitude').points,
standard_name='longitude',
units='degrees')
# create two cubes with lag btwn them but same coords.
lag_cube = Cube(lag.data,
dim_coords_and_dims=[(time, 0), (latitude, 1),
(longitude, 2)])
nolag_cube = Cube(nolag.data,
dim_coords_and_dims=[(time, 0), (latitude, 1),
(longitude, 2)])
# Calculate correlation
corr_cube = istats.pearsonr(lag_cube, nolag_cube, corr_coords='time')
return corr_cube
def cube_extract (cube, season):
t1 = PartialDateTime(year=1871, month=2)
t2 = PartialDateTime(year=2012, month=12)
# t1 = PartialDateTime(year=1948, month=2)
# t2 = PartialDateTime(year=2019, month=12)
constr_1 = iris.Constraint(time=lambda t: t1 < t.point < t2)
constr_2 = iris.Constraint(clim_season=season.lower())
cube_out = cube.extract(constr_1 & constr_2)
return cube_out
def download_monthly_precipitation():
retrieve(
variable="tp",
start=PartialDateTime(1990, 1, 1),
end=PartialDateTime(2019, 1, 1),
target_dir=os.path.join(DATA_DIR, "ERA5", "tp"),
monthly_mean=True,
download=True,
merge=True,
)
def monthly_averaging():
requests = retrieve(
variable=["2t", "10u", "10v"],
start=PartialDateTime(1990, 1, 1),
end=PartialDateTime(2019, 1, 1),
)
retrieval_processing(
requests,
n_threads=12,
delete_processed=True,
overwrite=False,
soft_filesize_limit=3,
)
def plot(dirname, field, vert_range, label):
with catch_warnings():
# SPEEDY output is not CF compliant
simplefilter('ignore', UserWarning)
# Load cubes
print(f'Plotting {field}')
analy = load_cube(f'{dirname}/mean.nc', field)
nature = load_cube('nature.nc', field)
sprd = load_cube(f'{dirname}/sprd.nc', field)
# Get minimum duration of data
time = min(
analy.coord('time').points[-1],
nature.coord('time').points[-1])
analy = analy.extract(Constraint(time=lambda t: t < time))
nature = nature.extract(Constraint(time=lambda t: t < time))
sprd = sprd.extract(Constraint(time=lambda t: t < time))
# Extract vertically over chosen vertical range
lev_constraint_lambda = lambda s: s <= vert_range[
0] and s > vert_range[1]
# RMSE
rmse = ((analy - nature)**2)\
.extract(Constraint(atmosphere_sigma_coordinate=lev_constraint_lambda))\
.collapsed(['atmosphere_sigma_coordinate', 'longitude'], MEAN)**0.5
# Spread
sprd = sprd.extract(Constraint(atmosphere_sigma_coordinate=lev_constraint_lambda))\
.collapsed(['atmosphere_sigma_coordinate', 'longitude'], MEAN)
# Compute time mean after March 1st 00:00
with FUTURE.context(cell_datetime_objects=True):
rmse = rmse.extract(Constraint(time=lambda t: t > PartialDateTime(month=3,day=1)))\
.collapsed('time', MEAN)
sprd = sprd.extract(Constraint(time=lambda t: t > PartialDateTime(month=3,day=1)))\
.collapsed('time', MEAN)
latitude_coord = rmse.coord('latitude')
rmse_h, = plt.plot(latitude_coord.points,
rmse.data,
label=f'{label} error',
linestyle='-')
sprd_h, = plt.plot(latitude_coord.points,
sprd.data,
label=f'{label} spread',
linestyle='--',
color=rmse_h.get_color())
return [rmse_h, sprd_h]
def extract_error(field, dirname):
analys = load_cube(f'{dirname}/anal_mean.nc', field)
nature = load_cube('nature.nc', field)
# Get minimum duration of data
time = min(analys.coord('time').cell(-1), nature.coord('time').cell(-1))
analys = analys.extract(Constraint(time=lambda t: t < time))
nature = nature.extract(Constraint(time=lambda t: t < time))
# Compute time mean after March 1st 00:00
analys = analys.extract(Constraint(time=lambda t: t > PartialDateTime(month=3,day=1)))
nature = nature.extract(Constraint(time=lambda t: t > PartialDateTime(month=3,day=1)))
return analys, nature, time
def cape_precipitation():
requests = retrieve(
variable=["cape", "tp"],
start=PartialDateTime(1990, 1, 1),
end=PartialDateTime(2019, 1, 1),
)
retrieval_processing(
requests,
processing_class=CAPEPrecipWorker,
n_threads=24,
delete_processed=True,
overwrite=False,
soft_filesize_limit=150,
)
def select_time(month_start: int, month_end: int, year_start: int, year_end: int, cube):
"""
Query data in the given time range.
Args:
month_start, month_end, year_start, year_end: input time range
cube: input data cube
Return: the cube after filtering
"""
month_start = PartialDateTime(month = month_start)
month_end = PartialDateTime(month = month_end)
year_start = PartialDateTime(year = year_start)
year_end = PartialDateTime(year = year_end)
part_temp = cube.extract(iris.Constraint(time=lambda x : month_start<=x<=month_end) & \
iris.Constraint(time=lambda x : year_start<=x<=year_end))
return part_temp
def test_year_month(self):
pdt = PartialDateTime(2016, 8)
actual = pdt2num(pdt, 'days since 2016-08-20', 'gregorian')
expected = -19.
self.assertEqual(actual, expected)
def test_date_time(self):
pdt = PartialDateTime(2016, 8, 22, 14, 42, 11)
actual = pdt2num(pdt, 'days since 2016-08-20', 'gregorian')
expected = 2.6126273148148148
assert_almost_equal(actual, expected)
def setUp(self):
"""Set up a test cube with several time points"""
cube = set_up_variable_cube(
np.ones((12, 12), dtype=np.float32),
time=datetime(2017, 2, 17, 6, 0),
frt=datetime(2017, 2, 17, 6, 0),
)
cube.remove_coord("forecast_period")
self.time_points = np.arange(1487311200, 1487354400,
3600).astype(np.int64)
self.cube = add_coordinate(
cube,
self.time_points,
"time",
dtype=np.int64,
coord_units="seconds since 1970-01-01 00:00:00",
)
self.time_dt = datetime(2017, 2, 17, 6, 0)
self.time_constraint = iris.Constraint(
time=lambda cell: cell.point == PartialDateTime(
self.time_dt.year,
self.time_dt.month,
self.time_dt.day,
self.time_dt.hour,
))
def test_full(self):
pd = PartialDateTime(*list(range(7)))
result = repr(pd)
self.assertEqual(
result, 'PartialDateTime(year=0, month=1, day=2,'
' hour=3, minute=4, second=5,'
' microsecond=6)')
def generate_year_constraint_with_window(year, window):
"""
generate a :class:`iris.Constraint` on the time axis
for specified year +/- window
Args:
* year (int):
centered year for the constraint
* window (int):
number of years around the given year
Returns:
an :class:`iris.Constraint` on the time-axis
"""
first_year = PartialDateTime(year=year - window)
last_year = PartialDateTime(year=year + window)
return Constraint(time=lambda cell: first_year <= cell.point <= last_year)
def test_PartialDateTime_unbounded_cell(self):
# Check that cell comparison works with PartialDateTimes.
dt = PartialDateTime(month=6)
cell = Cell(netcdftime.datetime(2010, 3, 1))
self.assertLess(cell, dt)
self.assertGreater(dt, cell)
self.assertLessEqual(cell, dt)
self.assertGreaterEqual(dt, cell)
def get_first_cube_datetimes(datasets):
"""Given a Datasets instance, get the datetimes associated with the first cube."""
datetimes = [
PartialDateTime(
year=datasets[0].cubes[0].coord("time").cell(i).point.year,
month=datasets[0].cubes[0].coord("time").cell(i).point.month,
) for i in range(datasets[0].cubes[0].shape[0])
]
return datetimes
def extract_spread(field, dirname, time):
analys = load_cube(f'{dirname}/anal_sprd.nc', field)
# Get minimum duration of data
analys = analys.extract(Constraint(time=lambda t: t < time))
# Compute time mean after March 1st 00:00
analys = analys.extract(Constraint(time=lambda t: t > PartialDateTime(month=3,day=1)))
return analys
def download_daily_precipitation():
requests = retrieve(
variable="tp",
start=PartialDateTime(1990, 1, 1),
end=PartialDateTime(2019, 1, 1),
target_dir=os.path.join(DATA_DIR, "ERA5", "tp_daily"),
monthly_mean=False,
download=False,
merge=False,
)
retrieval_processing(
requests,
processing_class=DailyAveragingWorker,
n_threads=24,
delete_processed=True,
overwrite=False,
soft_filesize_limit=250,
)
def test_cftime_interface(self):
# The `netcdf4` Python module introduced new calendar classes by v1.2.7
# This test is primarily of this interface, so the
# final test assertion is simple.
filename = tests.get_data_path(("PP", "structured", "small.pp"))
cube = iris.load_cube(filename)
pdt = PartialDateTime(year=1992, month=10, day=1, hour=2)
time_constraint = iris.Constraint(time=lambda cell: cell < pdt)
sub_cube = cube.extract(time_constraint)
self.assertEqual(sub_cube.coord("time").points.shape, (1, ))
def test_year_month_end_period_360_day(self):
pdt = PartialDateTime(2016, 8)
actual = pdt2num(pdt,
'days since 2016-08-20',
'360_day',
start_of_period=False)
expected = 10.
self.assertEqual(actual, expected)
def extract_error(field, dirname):
analys = load_cube(f'{dirname}/mean.nc', field)
nature = load_cube('nature.nc', field)
# Get minimum duration of data
time = min(
analys.coord('time').points[-1],
nature.coord('time').points[-1])
analys = analys.extract(Constraint(time=lambda t: t < time))
nature = nature.extract(Constraint(time=lambda t: t < time))
# Compute time mean after March 1st 00:00
with FUTURE.context(cell_datetime_objects=True):
analys = analys.extract(
Constraint(time=lambda t: t > PartialDateTime(month=3, day=1)))
nature = nature.extract(
Constraint(time=lambda t: t > PartialDateTime(month=3, day=1)))
return analys, nature, time
def test_PartialDateTime_bounded_cell(self):
# Check that bounded comparisions to a PartialDateTime
# raise an exception. These are not supported as they
# depend on the calendar.
dt = PartialDateTime(month=6)
cell = Cell(datetime.datetime(2010, 1, 1),
bound=[datetime.datetime(2010, 1, 1),
datetime.datetime(2011, 1, 1)])
self.assert_raises_on_comparison(cell, dt, TypeError,
'bounded region for datetime')
def test_available_data_files_with_invalid_time_extraction(self):
"""Test with files available and an attempted extraction of data at a
time that is not valid."""
diagnostic_name = 'temperature_on_height_levels'
msg = 'No diagnostics match .*'
time_extract = Constraint(time=PartialDateTime(2018, 1, 1, 0))
with self.assertRaisesRegexp(ValueError, msg):
get_additional_diagnostics(diagnostic_name, self.data_directory,
time_extract=time_extract)
def test_invalid_time(self, warning_list=None):
"""Case for a time that is unavailable within the diagnostic cube."""
plugin = extract_cube_at_time
time_dt = datetime.datetime(2017, 2, 18, 6, 0)
time_extract = iris.Constraint(time=PartialDateTime(
time_dt.year, time_dt.month, time_dt.day, time_dt.hour))
cubes = CubeList([self.cube])
plugin(cubes, time_dt, time_extract)
self.assertTrue(len(warning_list), 1)
self.assertTrue(issubclass(warning_list[0].category, UserWarning))
self.assertTrue("Forecast time" in str(warning_list[0]))
def test_netcdftime_interface(self):
# The `netcdf4` Python module introduced new calendar classes by v1.2.7
# This test is primarily of this interface, so the
# final test assertion is simple.
with iris.FUTURE.context(cell_datetime_objects=True):
filename = tests.get_data_path(('PP', 'structured', 'small.pp'))
cube = iris.load_cube(filename)
pdt = PartialDateTime(year=1992, month=10, day=1, hour=2)
time_constraint = iris.Constraint(time=lambda cell: cell < pdt)
sub_cube = cube.extract(time_constraint)
self.assertEqual(sub_cube.coord('time').points.shape, (1, ))
def _make_partial_date_time(date_string, frequency):
"""
Convert the fields in `date_string` into a PartialDateTime object. Formats
that are known about are:
YYYMM
YYYYMMDD
:param str date_string: The date string to process
:param str frequency: The frequency of data in the file
:returns: An Iris PartialDateTime object containing as much information as
could be deduced from date_string
:rtype: iris.time.PartialDateTime
:raises ValueError: If the string is not in a known format.
"""
if frequency in ('yr', 'dec'):
pdt = PartialDateTime(year=int(date_string[0:4]))
elif frequency == 'mon':
pdt = PartialDateTime(year=int(date_string[0:4]),
month=int(date_string[4:6]))
elif frequency == 'day':
pdt = PartialDateTime(year=int(date_string[0:4]),
month=int(date_string[4:6]),
day=int(date_string[6:8]))
elif frequency in ('6hr', '3hr', '1hr', 'hr'):
pdt = PartialDateTime(year=int(date_string[0:4]),
month=int(date_string[4:6]),
day=int(date_string[6:8]),
hour=int(date_string[8:10]),
minute=int(date_string[10:12]))
elif frequency == 'subhr':
pdt = PartialDateTime(year=int(date_string[0:4]),
month=int(date_string[4:6]),
day=int(date_string[6:8]),
hour=int(date_string[8:10]),
minute=int(date_string[10:12]),
second=int(date_string[12:14]))
else:
raise ValueError('Unsupported frequency string {}'.format(frequency))
return pdt
def test_high_freq_rounded(self):
self.metadata_high_freq['end_date'] = PartialDateTime(year=2014,
month=12,
day=22,
hour=12,
minute=1)
time_coord = self.cube.coord('time')
time_coord_points = time_coord.points.copy()
time_coord_points[-1] += 34 / 60**2
time_coord.points = time_coord_points
self.assertTrue(
_check_start_end_times(self.cube, self.metadata_high_freq))
def test_invalid_time(self, warning_list=None):
"""Case for a time that is unavailable within the diagnostic cube."""
plugin = extract_cube_at_time
time_dt = datetime(2017, 2, 18, 6, 0)
time_constraint = iris.Constraint(time=PartialDateTime(
time_dt.year, time_dt.month, time_dt.day, time_dt.hour))
cubes = CubeList([self.cube])
plugin(cubes, time_dt, time_constraint)
warning_msg = "Forecast time"
self.assertTrue(
any(item.category == UserWarning for item in warning_list))
self.assertTrue(any(warning_msg in str(item) for item in warning_list))
