def test_create_data_object_with_valid_datetime(self):
import datetime
from cis.time_util import convert_datetime_to_std_time
data = self.product().create_data_object([self.filename], True)
assert(data.coord('time').data[3] == convert_datetime_to_std_time(datetime.datetime(2012, 8, 25, 15, 32, 0o3)))
assert(data.coord('time').data[4] == convert_datetime_to_std_time(datetime.datetime(2012, 8, 26)))
def check_valid_min_max_args(min_val, max_val, step, parser, range_axis):
"""
If a val range was specified, checks that they are valid numbers and the min is less than the max
"""
from cis.parse_datetime import parse_as_number_or_datetime, parse_as_float_or_time_delta, \
parse_datetimestr_to_std_time
from cis.time_util import convert_datetime_to_std_time
import datetime
ax_range = {}
if min_val is not None:
dt = parse_as_number_or_datetime(min_val, range_axis + "min", parser)
if isinstance(dt, list):
ax_range[range_axis + "min"] = convert_datetime_to_std_time(datetime.datetime(*dt))
else:
ax_range[range_axis + "min"] = dt
if max_val is not None:
dt = parse_as_number_or_datetime(max_val, range_axis + "max", parser)
if isinstance(dt, list):
ax_range[range_axis + "max"] = convert_datetime_to_std_time(datetime.datetime(*dt))
else:
ax_range[range_axis + "max"] = dt
if step is not None:
ax_range[range_axis + "step"] = parse_as_float_or_time_delta(step, range_axis + "step", parser)
return ax_range
def test_can_subset_ungridded_data_by_time(self):
data = cis.test.util.mock.make_regular_4d_ungridded_data()
time_coord = data.coord('time')
constraint = subset_constraint.UngriddedSubsetConstraint()
constraint.set_limit(time_coord,
time_util.convert_datetime_to_std_time(datetime.datetime(1984, 8, 28)),
time_util.convert_datetime_to_std_time(datetime.datetime(1984, 8, 29)))
subset = constraint.constrain(data)
assert (subset.data.tolist() == [2, 3, 7, 8, 12, 13, 17, 18, 22, 23, 27, 28, 32, 33, 37, 38, 42, 43, 47, 48])
def test_that_can_calculate_mid_point_between_two_datetime(self):
from cis.time_util import calculate_mid_time
t1 = convert_datetime_to_std_time(dt.datetime(2010, 0o2, 0o5, 0, 0, 0))
t2 = convert_datetime_to_std_time(dt.datetime(2010, 0o2, 0o6, 0, 0, 0))
tm = calculate_mid_time(t1, t2)
eq_(
tm,
convert_datetime_to_std_time(dt.datetime(2010, 0o2, 0o5, 12, 0,
0)))
def test_can_subset_ungridded_data_by_time_altitude(self):
data = cis.test.util.mock.make_regular_4d_ungridded_data()
time_coord = data.coord('time')
alt_coord = data.coord('altitude')
constraint = subset_constraint.UngriddedSubsetConstraint()
constraint.set_limit(time_coord,
time_util.convert_datetime_to_std_time(datetime.datetime(1984, 8, 28)),
time_util.convert_datetime_to_std_time(datetime.datetime(1984, 8, 29)))
constraint.set_limit(alt_coord, 45.0, 75.0)
subset = constraint.constrain(data)
assert (subset.data.tolist() == [27, 28, 32, 33, 37, 38])
def __new__(cls,
lat=None,
lon=None,
alt=None,
pres=None,
t=None,
val=None):
"""
Small constructor for the HyperPoint named tuple to allow optional arguments
and set-up value list.
"""
# If no value was specified create an empty list, otherwise create a list with one entry
if val is None or val == []:
val = []
else:
val = [val]
# If t was given as a datetime we need to convert it into our standard time
if isinstance(t, datetime.datetime):
t = convert_datetime_to_std_time(t)
point = super(HyperPoint, cls).__new__(cls, lat, lon, alt, pres, t,
val)
# Store the coordinate tuple for this point in case we need it later
point.coord_tuple = point.get_coord_tuple()
return point
def convert_to_std_time(self, time_stamp_info=None):
"""
Convert this coordinate to standard time. It will use either: the units of the coordinate if it is a cf_units
Unit, or the first word of the units, combined with the time stamp (if the timestamp is not given an error is
thrown).
:param time_stamp_info: the time stamp info from the file, None if it does not exist
"""
from cis.time_util import convert_time_since_to_std_time, cis_standard_time_unit, \
convert_time_using_time_stamp_info_to_std_time, convert_datetime_to_std_time
from cf_units import Unit
if isinstance(self.units, Unit):
self._data = convert_time_since_to_std_time(self.data, self.units)
elif str(self.units).lower().startswith('datetime'):
self._data = convert_datetime_to_std_time(self.data)
else:
if time_stamp_info is None:
raise ValueError(
"File must have time stamp info if converting without 'since' in units definition"
)
self._data = convert_time_using_time_stamp_info_to_std_time(
self.data, self.units, time_stamp_info)
self.units = cis_standard_time_unit
def modified(self, lat=None, lon=None, alt=None, pres=None, t=None, val=None):
"""Creates a HyperPoint with modified values.
:param lat:
:param lon:
:param alt:
:param pres:
:param t:
:param val:
:return:
"""
values = [v for v in self]
if lat is not None:
values[HyperPoint.LATITUDE] = lat
if lon is not None:
values[HyperPoint.LONGITUDE] = lon
if alt is not None:
values[HyperPoint.ALTITUDE] = alt
if pres is not None:
values[HyperPoint.AIR_PRESSURE] = pres
if t is not None:
if isinstance(t, datetime.datetime):
values[HyperPoint.TIME] = convert_datetime_to_std_time(t)
if val is not None:
if val == []:
values[HyperPoint.VAL] = []
else:
values[HyperPoint.VAL] = [val]
point = super(HyperPoint, self).__new__(HyperPoint, *values)
point.coord_tuple = point.get_coord_tuple()
return point
def test_convert_julian_date_to_std_time(self):
import numpy as np
from cis.time_util import convert_datetime_to_std_time
julian_days = np.array([
2454637.8091, 2454637.8092, 2454637.8097, 2454637.8197,
2454638.8097, 2454657.8097, 2454737.8197, 2456638.8097,
2464657.8097
])
std_days = convert_julian_date_to_std_time(julian_days)
ref = convert_datetime_to_std_time([
dt.datetime(2008, 6, 20, 7, 25, 6),
dt.datetime(2008, 6, 20, 7, 25, 15),
dt.datetime(2008, 6, 20, 7, 25, 58),
dt.datetime(2008, 6, 20, 7, 40, 22),
dt.datetime(2008, 6, 21, 7, 25, 58),
dt.datetime(2008, 7, 10, 7, 25, 58),
dt.datetime(2008, 9, 28, 7, 40, 22),
dt.datetime(2013, 12, 12, 7, 25, 58),
dt.datetime(2035, 11, 26, 7, 25, 58)
])
eq_(julian_days.shape, std_days.shape)
assert np.allclose(std_days, ref)
def test_GIVEN_single_point_in_cube_WHEN_iterate_THEN_return_point_in_middle(self):
sample_cube = make_square_5x3_2d_cube_with_time(offset=0, time_offset=0)
data_point = make_dummy_ungridded_data_single_point(0.5, 0.5, 1.2, time=datetime.datetime(1984, 8, 28, 0, 0))
coord_map = make_coord_map(sample_cube, data_point)
coords = sample_cube.coords()
for (hpi, ci, shi) in coord_map:
coord = coords[ci]
if coord.ndim > 1:
raise NotImplementedError("Co-location of data onto a cube with a coordinate of dimension greater"
" than one is not supported (coordinate %s)", coord.name())
# Ensure that bounds exist.
if not coord.has_bounds():
coord.guess_bounds()
constraint = BinnedCubeCellOnlyConstraint()
data_index.create_indexes(constraint, coords, data_point.get_non_masked_points(), coord_map)
iterator = constraint.get_iterator(False, coord_map, coords, data_point.get_non_masked_points(), None,
sample_cube, None)
final_points_index = [(out_index, hp, points) for out_index, hp, points in iterator]
assert_that(len(final_points_index), is_(1), "There is one mapping from sample_cube to the final grid")
assert_that(final_points_index[0][0], is_((2, 1, 1)), "The points should map to index")
assert_that(final_points_index[0][1], is_(HyperPoint(lat=0, lon=0, t=datetime.datetime(1984, 8, 28))),
"The points should map to index")
assert_that(final_points_index[0][2].latitudes, is_([0.5]), "The points should map to index")
assert_that(final_points_index[0][2].longitudes, is_([0.5]), "The points should map to index")
assert_that(final_points_index[0][2].times,
is_([convert_datetime_to_std_time(datetime.datetime(1984, 8, 28, 0, 0))]),
"The points should map to index")
assert_that(final_points_index[0][2].vals, is_([1.2]), "The points should map to index")
def test_change_ug_data_year(self):
from cis.test.util.mock import make_regular_2d_with_time_ungridded_data
from cis.time_util import convert_datetime_to_std_time
from datetime import datetime
ug = make_regular_2d_with_time_ungridded_data()
change_year_of_ungridded_data(ug, 2007)
eq_(
ug.coord('time').points[0, 0],
convert_datetime_to_std_time(datetime(2007, 8, 27)))
def make_square_5x3_2d_cube_with_scalar_time(time_offset=0):
"""
Makes a well defined cube of shape 5x3 with data as follows
array([[1,2,3],
[4,5,6],
[7,8,9],
[10,11,12],
[13,14,15]])
and coordinates in latitude:
array([ -10, -5, 0, 5, 10 ])
longitude:
array([ -5, 0, 5 ])
time:
1984-08-27
time_bounds:
[1984-08-22, 1984-09-01]
"""
import numpy as np
from iris.cube import Cube
from iris.coords import DimCoord
from cis.time_util import cis_standard_time_unit
t0 = datetime.datetime(1984, 8, 27)
time_nums = convert_datetime_to_std_time(t0 + datetime.timedelta(days=time_offset))
time = DimCoord(time_nums, standard_name='time',
bounds=[convert_datetime_to_std_time(t0 - datetime.timedelta(days=5)),
convert_datetime_to_std_time(t0 + datetime.timedelta(days=5))],
units=cis_standard_time_unit)
latitude = DimCoord(np.arange(-10., 11., 5), var_name='lat', standard_name='latitude', units='degrees')
longitude = DimCoord(np.arange(-5., 6., 5), var_name='lon', standard_name='longitude', units='degrees')
data = np.reshape(np.arange(15) + 1.0, (5, 3))
cube = Cube(data, dim_coords_and_dims=[(latitude, 0), (longitude, 1)], var_name='dummy')
cube.add_aux_coord(time)
return cube
def make_square_NxM_2d_cube_with_time(start_lat=-10, end_lat=10, lat_point_count=5,
start_lon=-5, end_lon=5, lon_point_count=3,
time_offset=0):
"""
Makes a well defined cube of shape 5x3 with data as follows
arr([[[ 1. 2. 3. 4. 5. 6. 7.]
[ 8. 9. 10. 11. 12. 13. 14.]
[ 15. 16. 17. 18. 19. 20. 21.]]
[[ 22. 23. 24. 25. 26. 27. 28.]
[ 29. 30. 31. 32. 33. 34. 35.]
[ 36. 37. 38. 39. 40. 41. 42.]]
[[ 43. 44. 45. 46. 47. 48. 49.]
[ 50. 51. 52. 53. 54. 55. 56.]
[ 57. 58. 59. 60. 61. 62. 63.]]
[[ 64. 65. 66. 67. 68. 69. 70.]
[ 71. 72. 73. 74. 75. 76. 77.]
[ 78. 79. 80. 81. 82. 83. 84.]]
[[ 85. 86. 87. 88. 89. 90. 91.]
[ 92. 93. 94. 95. 96. 97. 98.]
[ 99. 100. 101. 102. 103. 104. 105.]]])
and coordinates in latitude:
array([ -10, -5, 0, 5, 10 ])
longitude:
array([ -5, 0, 5 ])
time:
array([1984-08-27, 1984-08-28, 1984-08-29, 1984-08-30, 1984-08-31, 1984-09-01, 1984-09-02])
They are different lengths to make it easier to distinguish. Note the latitude increases
as you step through the array in order - so downwards as it's written above
"""
import numpy as np
from iris.cube import Cube
from iris.coords import DimCoord
import datetime
t0 = datetime.datetime(1984, 8, 27)
times = np.array([t0 + datetime.timedelta(days=d + time_offset) for d in range(7)])
time_nums = convert_datetime_to_std_time(times)
time = DimCoord(time_nums, standard_name='time')
latitude = DimCoord(np.linspace(start_lat, end_lat, lat_point_count), standard_name='latitude', units='degrees')
longitude = DimCoord(np.linspace(start_lon, end_lon, lon_point_count), standard_name='longitude', units='degrees')
data = np.reshape(np.arange(lat_point_count * lon_point_count * 7) + 1.0, (lat_point_count, lon_point_count, 7))
cube = Cube(data, dim_coords_and_dims=[(latitude, 0), (longitude, 1), (time, 2)], var_name='dummy')
return cube
def test_GIVEN_single_point_in_cube_WHEN_iterate_THEN_return_point_in_middle(
self):
sample_cube = make_square_5x3_2d_cube_with_time(offset=0,
time_offset=0)
data_point = make_dummy_ungridded_data_single_point(
0.5, 0.5, 1.2, time=datetime.datetime(1984, 8, 28, 0, 0))
coord_map = make_coord_map(sample_cube, data_point)
coords = sample_cube.coords()
for (hpi, ci, shi) in coord_map:
coord = coords[ci]
if coord.ndim > 1:
raise NotImplementedError(
"Co-location of data onto a cube with a coordinate of dimension greater"
" than one is not supported (coordinate %s)", coord.name())
# Ensure that bounds exist.
if not coord.has_bounds():
coord.guess_bounds()
constraint = BinnedCubeCellOnlyConstraint()
data_index.create_indexes(constraint, coords,
data_point.get_non_masked_points(),
coord_map)
iterator = constraint.get_iterator(False, coord_map, coords,
data_point.get_non_masked_points(),
None, sample_cube, None)
final_points_index = [(out_index, hp, points)
for out_index, hp, points in iterator]
assert_that(len(final_points_index), is_(1),
"There is one mapping from sample_cube to the final grid")
assert_that(final_points_index[0][0], is_((2, 1, 1)),
"The points should map to index")
assert_that(
final_points_index[0][1],
is_(HyperPoint(lat=0, lon=0, t=datetime.datetime(1984, 8, 28))),
"The points should map to index")
assert_that(final_points_index[0][2].latitudes, is_([0.5]),
"The points should map to index")
assert_that(final_points_index[0][2].longitudes, is_([0.5]),
"The points should map to index")
assert_that(
final_points_index[0][2].times,
is_([
convert_datetime_to_std_time(
datetime.datetime(1984, 8, 28, 0, 0))
]), "The points should map to index")
assert_that(final_points_index[0][2].vals, is_([1.2]),
"The points should map to index")
def create_coords(self, filenames, variable=None):
from cis.data_io.ungridded_data import Metadata
from numpy import genfromtxt, NaN
from cis.exceptions import InvalidVariableError
from cis.time_util import convert_datetime_to_std_time
import dateutil.parser as du
array_list = []
for filename in filenames:
try:
array_list.append(genfromtxt(filename, dtype="f8,f8,f8,O,f8",
names=['latitude', 'longitude', 'altitude', 'time', 'value'],
delimiter=',', missing_values='', usemask=True, invalid_raise=True,
converters={"time": du.parse}))
except:
raise IOError('Unable to read file ' + filename)
data_array = utils.concatenate(array_list)
n_elements = len(data_array['latitude'])
coords = CoordList()
coords.append(Coord(data_array["latitude"],
Metadata(standard_name="latitude", shape=(n_elements,), units="degrees_north")))
coords.append(Coord(data_array["longitude"],
Metadata(standard_name="longitude", shape=(n_elements,), units="degrees_east")))
coords.append(
Coord(data_array["altitude"], Metadata(standard_name="altitude", shape=(n_elements,), units="meters")))
time_arr = convert_datetime_to_std_time(data_array["time"])
time = Coord(time_arr,
Metadata(standard_name="time", shape=(n_elements,), units="days since 1600-01-01 00:00:00"))
coords.append(time)
if variable:
try:
data = UngriddedData(data_array['value'], Metadata(name="value", shape=(n_elements,), units="unknown",
missing_value=NaN), coords)
except:
InvalidVariableError("Value column does not exist in file " + filenames)
return data
else:
return UngriddedCoordinates(coords)
def __new__(cls, lat=None, lon=None, alt=None, pres=None, t=None, val=None):
"""
Small constructor for the HyperPoint named tuple to allow optional arguments
and set-up value list.
"""
# If no value was specified create an empty list, otherwise create a list with one entry
if val is None or val == []:
val = []
else:
val = [val]
# If t was given as a datetime we need to convert it into our standard time
if isinstance(t, datetime.datetime):
t = convert_datetime_to_std_time(t)
point = super(HyperPoint, cls).__new__(cls, lat, lon, alt, pres, t, val)
# Store the coordinate tuple for this point in case we need it later
point.coord_tuple = point.get_coord_tuple()
return point
def test_convert_julian_date_to_std_time(self):
import numpy as np
from cis.time_util import convert_datetime_to_std_time
julian_days = np.array([2454637.8091, 2454637.8092, 2454637.8097,
2454637.8197, 2454638.8097, 2454657.8097,
2454737.8197, 2456638.8097, 2464657.8097])
std_days = convert_julian_date_to_std_time(julian_days)
ref = convert_datetime_to_std_time([dt.datetime(2008, 6, 20, 7, 25, 6),
dt.datetime(2008, 6, 20, 7, 25, 15),
dt.datetime(2008, 6, 20, 7, 25, 58),
dt.datetime(2008, 6, 20, 7, 40, 22),
dt.datetime(2008, 6, 21, 7, 25, 58),
dt.datetime(2008, 7, 10, 7, 25, 58),
dt.datetime(2008, 9, 28, 7, 40, 22),
dt.datetime(2013, 12, 12, 7, 25, 58),
dt.datetime(2035, 11, 26, 7, 25, 58)])
eq_(julian_days.shape, std_days.shape)
assert np.allclose(std_days, ref)
def convert_to_std_time(self, time_stamp_info=None):
"""
Convert this coordinate to standard time. It will use either: the units of the coordinate if it is a cf_units
Unit, or the first word of the units, combined with the time stamp (if the timestamp is not given an error is
thrown).
:param time_stamp_info: the time stamp info from the file, None if it does not exist
"""
from cis.time_util import convert_time_since_to_std_time, cis_standard_time_unit, \
convert_time_using_time_stamp_info_to_std_time, convert_datetime_to_std_time
from cf_units import Unit
if isinstance(self.units, Unit):
self._data = convert_time_since_to_std_time(self.data, self.units)
elif str(self.units).lower().startswith('datetime'):
self._data = convert_datetime_to_std_time(self.data)
else:
if time_stamp_info is None:
raise ValueError("File must have time stamp info if converting without 'since' in units definition")
self._data = convert_time_using_time_stamp_info_to_std_time(self.data, self.units, time_stamp_info)
self.units = cis_standard_time_unit
def test_that_can_parse_datetimestr_to_obj():
from cis.time_util import convert_datetime_to_std_time
import datetime as dt
# when not specifying the hours, minutes or seconds, 0 is used
eq_(parse_datetimestr_to_std_time("2010-02-05 02:15:45"),
convert_datetime_to_std_time(dt.datetime(2010, 2, 5, 2, 15, 45)))
eq_(parse_datetimestr_to_std_time("2010-02-05 02:15"),
convert_datetime_to_std_time(dt.datetime(2010, 2, 5, 2, 15, 0)))
eq_(parse_datetimestr_to_std_time("2010-02-05 02"),
convert_datetime_to_std_time(dt.datetime(2010, 2, 5, 2, 0, 0)))
eq_(parse_datetimestr_to_std_time("2010-02-05"),
convert_datetime_to_std_time(dt.datetime(2010, 2, 5, 0, 0, 0)))
# GOTCHA: when not specifying an element of a date (i.e. the year, month or day), the current date is used
now = dt.datetime.now()
eq_(parse_datetimestr_to_std_time("2010-02-05"),
convert_datetime_to_std_time(dt.datetime(2010, 2, 5)))
eq_(parse_datetimestr_to_std_time("2010-12"),
convert_datetime_to_std_time(dt.datetime(2010, 12, now.day)))
eq_(parse_datetimestr_to_std_time("2010-"),
convert_datetime_to_std_time(dt.datetime(2010, now.month, now.day)))
def modified(self,
lat=None,
lon=None,
alt=None,
pres=None,
t=None,
val=None):
"""Creates a HyperPoint with modified values.
:param lat:
:param lon:
:param alt:
:param pres:
:param t:
:param val:
:return:
"""
values = [v for v in self]
if lat is not None:
values[HyperPoint.LATITUDE] = lat
if lon is not None:
values[HyperPoint.LONGITUDE] = lon
if alt is not None:
values[HyperPoint.ALTITUDE] = alt
if pres is not None:
values[HyperPoint.AIR_PRESSURE] = pres
if t is not None:
if isinstance(t, datetime.datetime):
values[HyperPoint.TIME] = convert_datetime_to_std_time(t)
if val is not None:
if val == []:
values[HyperPoint.VAL] = []
else:
values[HyperPoint.VAL] = [val]
point = super(HyperPoint, self).__new__(HyperPoint, *values)
point.coord_tuple = point.get_coord_tuple()
return point
def index_data(self, coords, hyper_points, coord_map):
"""
Index the data that falls inside the grid cells
:param coords: coordinates of grid
:param hyper_points: list of HyperPoints to index
:param coord_map: list of tuples relating index in HyperPoint to index in coords and in
coords to be iterated over
"""
# create bounds in correct order
hp_coords = []
coord_descreasing = [False] * len(coords)
coord_lengths = [0] * len(coords)
lower_bounds = [None] * len(coords)
max_bounds = [None] * len(coords)
for (hpi, ci, shi) in coord_map:
coord = coords[ci]
# Coordinates must be monotonic; determine whether increasing or decreasing.
if len(coord.points) > 1:
if coord.points[1] < coord.points[0]:
coord_descreasing[shi] = True
coord_lengths[shi] = len(coord.points)
if coord_descreasing[shi]:
lower_bounds[shi] = coord.bounds[::-1, 1]
max_bounds[shi] = coord.bounds[0, 1]
else:
lower_bounds[shi] = coord.bounds[::, 0]
max_bounds[shi] = coord.bounds[-1, 1]
hp_coord = hyper_points.coords[hpi]
if isinstance(hp_coord[0], datetime.datetime):
hp_coord = convert_datetime_to_std_time(hp_coord)
hp_coords.append(hp_coord)
bounds_coords_max = list(zip(lower_bounds, hp_coords, max_bounds))
# stack for each coordinate
# where the coordinate is larger than the maximum set to -1
# otherwise search in the sorted coordinate to find all the index of the hyperpoints
# The choice of 'left' or 'right' and '<' and '<=' determines which
# cell is chosen when the coordinate is equal to the boundary.
# -1 or M_i indicates the point is outside the grid.
# Output is a list of coordinates which lists the indexes where the hyper points
# should be located in the grid
indices = np.vstack(
np.where(ci < max_coordinate_value,
np.searchsorted(bi, ci, side='right') - 1, -1)
for bi, ci, max_coordinate_value in bounds_coords_max)
# D-tuple giving the shape of the output grid
grid_shape = tuple(len(bi_ci[0]) for bi_ci in bounds_coords_max)
# shape (N,) telling which points actually fall within the grid,
# i.e. have indexes that are not -1 and are not masked data points
grid_mask = np.all(
(indices >= 0) & (ma.getmaskarray(hyper_points.data) == False),
axis=0)
# if the coordinate was decreasing then correct the indices for this cell
for indices_slice, decreasing, coord_length in zip(
range(indices.shape[0]), coord_descreasing, coord_lengths):
if decreasing:
# indices[indices_slice] += (coord_length - 1) - indices[indices_slice]
indices[indices_slice] *= -1
indices[indices_slice] += (coord_length - 1)
# shape (N,) containing negative scalar cell numbers for each
# input point (sequence doesn't matter so long as they are unique), or
# -1 for points outside the grid.
#
# Possibly numpy.lexsort could be used to avoid the need for this,
# although we'd have to be careful about points outside the grid.
self.cell_numbers = np.where(
grid_mask,
np.tensordot(np.cumproduct((1, ) + grid_shape[:-1]),
indices,
axes=1), -1)
# Sort everything by cell number
self.sort_order = np.argsort(self.cell_numbers)
self.cell_numbers = self.cell_numbers[self.sort_order]
self._indices = indices[:, self.sort_order]
self.hp_coords = [hp_coord[self.sort_order] for hp_coord in hp_coords]
def convert_datetime_to_standard_time(self):
from cis.time_util import convert_datetime_to_std_time, cis_standard_time_unit
self._data = convert_datetime_to_std_time(self.data)
self.units = cis_standard_time_unit
def convert_datetime_to_standard_time(self):
from cis.time_util import convert_datetime_to_std_time, cis_standard_time_unit
self._data = convert_datetime_to_std_time(self.data)
self.units = str(cis_standard_time_unit)
self.metadata.calendar = cis_standard_time_unit.calendar
def convert_datetime_to_standard_time(self):
from cis.time_util import convert_datetime_to_std_time, cis_standard_time_unit
self._data = convert_datetime_to_std_time(self.data)
self.units = cis_standard_time_unit
def test_that_can_calculate_mid_point_between_two_datetime():
from cis.time_util import calculate_mid_time
t1 = convert_datetime_to_std_time(dt.datetime(2010, 02, 05, 0, 0, 0))
t2 = convert_datetime_to_std_time(dt.datetime(2010, 02, 06, 0, 0, 0))
tm = calculate_mid_time(t1, t2)
eq_(tm, convert_datetime_to_std_time(dt.datetime(2010, 02, 05, 12, 0, 0)))
def make_mock_cube(lat_dim_length=5, lon_dim_length=3, lon_range=None, alt_dim_length=0, pres_dim_length=0,
time_dim_length=0,
horizontal_offset=0, altitude_offset=0, pressure_offset=0, time_offset=0, data_offset=0,
surf_pres_offset=0,
hybrid_ht_len=0, hybrid_pr_len=0, geopotential_height=False, dim_order=None, mask=False):
"""
Makes a cube of any shape required, with coordinate offsets from the default available. If no arguments are
given get a 5x3 cube of the form:
array([[1,2,3],
[4,5,6],
[7,8,9],
[10,11,12],
[13,14,15]])
and coordinates in latitude:
array([ -10, -5, 0, 5, 10 ])
longitude:
array([ -5, 0, 5 ])
:param lat_dim_length: Latitude grid length
:param lon_dim_length: Longitude grid length
:param alt_dim_length: Altitude grid length
:param pres_dim_length: Pressure grid length
:param time_dim_length: Time grid length
:param horizontal_offset: Offset from the default grid, in degrees, in lat and lon
:param altitude_offset: Offset from the default grid in altitude
:param pressure_offset: Offset from the default grid in pressure
:param time_offset: Offset from the default grid in time
:param data_offset: Offset from the default data values
:param surf_pres_offset: Offset for the optional surface pressure field
:param hybrid_ht_len: Hybrid height grid length
:param hybrid_pr_len: Hybrid pressure grid length
:param geopotential_height: Include a geopotential height field when calcluting a hybrid pressure? (default False)
:param dim_order: List of 'lat', 'lon', 'alt', 'pres', 'time' in the order in which the dimensions occur
:param mask: A mask to apply to the data, this should be either a scalar or the same shape as the data
:return: A cube with well defined data.
"""
import iris
from iris.aux_factory import HybridHeightFactory, HybridPressureFactory
data_size = 1
DIM_NAMES = ['lat', 'lon', 'alt', 'pres', 'time', 'hybrid_ht', 'hybrid_pr']
dim_lengths = [lat_dim_length, lon_dim_length, alt_dim_length, pres_dim_length, time_dim_length, hybrid_ht_len,
hybrid_pr_len]
lon_range = lon_range or (-5., 5.)
if dim_order is None:
dim_order = list(DIM_NAMES)
if any([True for d in dim_order if d not in DIM_NAMES]):
raise ValueError("dim_order contains unrecognised name")
for idx, dim in enumerate(DIM_NAMES):
if dim_lengths[idx] == 0 and dim in dim_order:
del dim_order[dim_order.index(dim)]
coord_map = {}
for idx, dim in enumerate(dim_order):
coord_map[dim] = dim_order.index(dim)
coord_list = [None] * len(coord_map)
if lat_dim_length:
coord_list[coord_map['lat']] = (DimCoord(np.linspace(-10., 10., lat_dim_length) + horizontal_offset,
standard_name='latitude', units='degrees', var_name='lat'),
coord_map['lat'])
data_size *= lat_dim_length
if lon_dim_length:
coord_list[coord_map['lon']] = (
DimCoord(np.linspace(lon_range[0], lon_range[1], lon_dim_length) + horizontal_offset,
standard_name='longitude', units='degrees', var_name='lon'), coord_map['lon'])
data_size *= lon_dim_length
if alt_dim_length:
coord_list[coord_map['alt']] = (DimCoord(np.linspace(0., 7., alt_dim_length) + altitude_offset,
standard_name='altitude', units='metres', var_name='alt'),
coord_map['alt'])
data_size *= alt_dim_length
if pres_dim_length:
coord_list[coord_map['pres']] = (DimCoord(np.linspace(0., 7., pres_dim_length) + pressure_offset,
standard_name='air_pressure', units='hPa', var_name='pres'),
coord_map['pres'])
data_size *= pres_dim_length
if time_dim_length:
t0 = datetime.datetime(1984, 8, 27)
times = np.array([t0 + datetime.timedelta(days=d + time_offset) for d in range(time_dim_length)])
time_nums = convert_datetime_to_std_time(times)
time_bounds = None
if time_dim_length == 1:
time_bounds = convert_datetime_to_std_time(np.array([times[0] - datetime.timedelta(days=0.5),
times[0] + datetime.timedelta(days=0.5)]))
coord_list[coord_map['time']] = (DimCoord(time_nums, standard_name='time',
units='days since 1600-01-01 00:00:00', var_name='time',
bounds=time_bounds),
coord_map['time'])
data_size *= time_dim_length
if hybrid_ht_len:
coord_list[coord_map['hybrid_ht']] = (DimCoord(np.arange(hybrid_ht_len, dtype='i8') + 10,
"model_level_number", units="1"), coord_map['hybrid_ht'])
data_size *= hybrid_ht_len
if hybrid_pr_len:
coord_list[coord_map['hybrid_pr']] = (DimCoord(np.arange(hybrid_pr_len, dtype='i8'),
"atmosphere_hybrid_sigma_pressure_coordinate", units="1"),
coord_map['hybrid_pr'])
data_size *= hybrid_pr_len
data = np.reshape(np.arange(data_size) + data_offset + 1., tuple(len(i[0].points) for i in coord_list))
if mask:
data = np.ma.asarray(data)
data.mask = mask
return_cube = Cube(data, dim_coords_and_dims=coord_list, var_name='rain', standard_name='rainfall_rate',
long_name="TOTAL RAINFALL RATE: LS+CONV KG/M2/S", units="kg m-2 s-1")
if hybrid_ht_len:
return_cube.add_aux_coord(iris.coords.AuxCoord(np.arange(hybrid_ht_len, dtype='i8') + 40,
long_name="level_height",
units="m", var_name='hybrid_ht'), coord_map['hybrid_ht'])
return_cube.add_aux_coord(iris.coords.AuxCoord(np.arange(hybrid_ht_len, dtype='i8') + 50,
long_name="sigma", units="1", var_name='sigma'),
coord_map['hybrid_ht'])
return_cube.add_aux_coord(iris.coords.AuxCoord(
np.arange(lat_dim_length * lon_dim_length, dtype='i8').reshape(lat_dim_length, lon_dim_length) + 100,
long_name="surface_altitude",
units="m"), [coord_map['lat'], coord_map['lon']])
return_cube.add_aux_factory(HybridHeightFactory(
delta=return_cube.coord("level_height"),
sigma=return_cube.coord("sigma"),
orography=return_cube.coord("surface_altitude")))
elif hybrid_pr_len:
return_cube.add_aux_coord(iris.coords.AuxCoord(np.arange(hybrid_pr_len, dtype='i8') + 40,
long_name="hybrid A coefficient at layer midpoints",
units="Pa", var_name='a'), coord_map['hybrid_pr'])
return_cube.add_aux_coord(iris.coords.AuxCoord(np.arange(hybrid_pr_len, dtype='f8') + 50,
long_name="hybrid B coefficient at layer midpoints", units="1",
var_name='b'),
coord_map['hybrid_pr'])
return_cube.add_aux_coord(
iris.coords.AuxCoord(np.arange(lat_dim_length * lon_dim_length * time_dim_length, dtype='i8')
.reshape(lat_dim_length, lon_dim_length, time_dim_length) * 100000 + surf_pres_offset,
"surface_air_pressure", units="Pa"),
[coord_map['lat'], coord_map['lon'], coord_map['time']])
if geopotential_height:
return_cube.add_aux_coord(iris.coords.AuxCoord(
np.arange(lat_dim_length * lon_dim_length * time_dim_length * hybrid_pr_len, dtype='i8')
.reshape(lat_dim_length, lon_dim_length, time_dim_length, hybrid_pr_len) + 10,
"altitude", long_name="Geopotential height at layer midpoints", units="meter"),
[coord_map['lat'], coord_map['lon'], coord_map['time'], coord_map['hybrid_pr']])
return_cube.add_aux_factory(HybridPressureFactory(
delta=return_cube.coord("hybrid A coefficient at layer midpoints"),
sigma=return_cube.coord("hybrid B coefficient at layer midpoints"),
surface_air_pressure=return_cube.coord("surface_air_pressure")))
for coord in return_cube.coords(dim_coords=True):
if coord.bounds is None:
coord.guess_bounds()
return return_cube
def index_data(self, coords, hyper_points, coord_map):
"""
Index the data that falls inside the grid cells
:param coords: coordinates of grid
:param hyper_points: list of HyperPoints to index
:param coord_map: list of tuples relating index in HyperPoint to index in coords and in
coords to be iterated over
"""
# create bounds in correct order
hp_coords = []
coord_descreasing = [False] * len(coords)
coord_lengths = [0] * len(coords)
lower_bounds = [None] * len(coords)
max_bounds = [None] * len(coords)
for (hpi, ci, shi) in coord_map:
coord = coords[ci]
# Coordinates must be monotonic; determine whether increasing or decreasing.
if len(coord.points) > 1:
if coord.points[1] < coord.points[0]:
coord_descreasing[shi] = True
coord_lengths[shi] = len(coord.points)
if coord_descreasing[shi]:
lower_bounds[shi] = coord.bounds[::-1, 1]
max_bounds[shi] = coord.bounds[0, 1]
else:
lower_bounds[shi] = coord.bounds[::, 0]
max_bounds[shi] = coord.bounds[-1, 1]
hp_coord = hyper_points.coords[hpi]
if isinstance(hp_coord[0], datetime.datetime):
hp_coord = convert_datetime_to_std_time(hp_coord)
hp_coords.append(hp_coord)
bounds_coords_max = list(zip(lower_bounds, hp_coords, max_bounds))
# stack for each coordinate
# where the coordinate is larger than the maximum set to -1
# otherwise search in the sorted coordinate to find all the index of the hyperpoints
# The choice of 'left' or 'right' and '<' and '<=' determines which
# cell is chosen when the coordinate is equal to the boundary.
# -1 or M_i indicates the point is outside the grid.
# Output is a list of coordinates which lists the indexes where the hyper points
# should be located in the grid
indices = np.vstack(
np.where(
ci < max_coordinate_value,
np.searchsorted(bi, ci, side='right') - 1,
-1)
for bi, ci, max_coordinate_value in bounds_coords_max)
# D-tuple giving the shape of the output grid
grid_shape = tuple(len(bi_ci[0]) for bi_ci in bounds_coords_max)
# shape (N,) telling which points actually fall within the grid,
# i.e. have indexes that are not -1 and are not masked data points
grid_mask = np.all(
(indices >= 0) &
(ma.getmaskarray(hyper_points.data) == False),
axis=0)
# if the coordinate was decreasing then correct the indices for this cell
for indices_slice, decreasing, coord_length in zip(range(indices.shape[0]), coord_descreasing, coord_lengths):
if decreasing:
# indices[indices_slice] += (coord_length - 1) - indices[indices_slice]
indices[indices_slice] *= -1
indices[indices_slice] += (coord_length - 1)
# shape (N,) containing negative scalar cell numbers for each
# input point (sequence doesn't matter so long as they are unique), or
# -1 for points outside the grid.
#
# Possibly numpy.lexsort could be used to avoid the need for this,
# although we'd have to be careful about points outside the grid.
self.cell_numbers = np.where(
grid_mask,
np.tensordot(
np.cumproduct((1,) + grid_shape[:-1]),
indices,
axes=1
),
-1)
# Sort everything by cell number
self.sort_order = np.argsort(self.cell_numbers)
self.cell_numbers = self.cell_numbers[self.sort_order]
self._indices = indices[:, self.sort_order]
self.hp_coords = [hp_coord[self.sort_order] for hp_coord in hp_coords]
def make_regular_4d_ungridded_data():
"""
Makes a well defined ungridded data object of shape 10x5 with data as follows
data:
[[ 1. 2. 3. 4. 5.]
[ 6. 7. 8. 9. 10.]
[ 11. 12. 13. 14. 15.]
[ 16. 17. 18. 19. 20.]
[ 21. 22. 23. 24. 25.]
[ 26. 27. 28. 29. 30.]
[ 31. 32. 33. 34. 35.]
[ 36. 37. 38. 39. 40.]
[ 41. 42. 43. 44. 45.]
[ 46. 47. 48. 49. 50.]]
latitude:
[[-10. -5. 0. 5. 10.]
[-10. -5. 0. 5. 10.]
[-10. -5. 0. 5. 10.]
[-10. -5. 0. 5. 10.]
[-10. -5. 0. 5. 10.]
[-10. -5. 0. 5. 10.]
[-10. -5. 0. 5. 10.]
[-10. -5. 0. 5. 10.]
[-10. -5. 0. 5. 10.]
[-10. -5. 0. 5. 10.]]
longitude:
[[-5. -2.5 0. 2.5 5. ]
[-5. -2.5 0. 2.5 5. ]
[-5. -2.5 0. 2.5 5. ]
[-5. -2.5 0. 2.5 5. ]
[-5. -2.5 0. 2.5 5. ]
[-5. -2.5 0. 2.5 5. ]
[-5. -2.5 0. 2.5 5. ]
[-5. -2.5 0. 2.5 5. ]
[-5. -2.5 0. 2.5 5. ]
[-5. -2.5 0. 2.5 5. ]]
altitude:
[[ 0. 0. 0. 0. 0.]
[ 10. 10. 10. 10. 10.]
[ 20. 20. 20. 20. 20.]
[ 30. 30. 30. 30. 30.]
[ 40. 40. 40. 40. 40.]
[ 50. 50. 50. 50. 50.]
[ 60. 60. 60. 60. 60.]
[ 70. 70. 70. 70. 70.]
[ 80. 80. 80. 80. 80.]
[ 90. 90. 90. 90. 90.]]
pressure:
[[ 4. 4. 4. 4. 4.]
[ 16. 16. 16. 16. 16.]
[ 20. 20. 20. 20. 20.]
[ 30. 30. 30. 30. 30.]
[ 40. 40. 40. 40. 40.]
[ 50. 50. 50. 50. 50.]
[ 60. 60. 60. 60. 60.]
[ 70. 70. 70. 70. 70.]
[ 80. 80. 80. 80. 80.]
[ 90. 90. 90. 90. 90.]]
time:
[[1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31]
[1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31]
[1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31]
[1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31]
[1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31]
[1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31]
[1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31]
[1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31]
[1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31]
[1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31]]
They are shaped to represent a typical lidar type satelite data set.
"""
import numpy as np
from cis.data_io.Coord import CoordList, Coord
from cis.data_io.ungridded_data import UngriddedData, Metadata
from cis.time_util import cis_standard_time_unit
import datetime
x_points = np.linspace(-10, 10, 5)
y_points = np.linspace(-5, 5, 5)
t0 = datetime.datetime(1984, 8, 27)
times = convert_datetime_to_std_time(np.array([t0 + datetime.timedelta(days=d) for d in range(5)]))
alt = np.linspace(0, 90, 10)
data = np.reshape(np.arange(50) + 1.0, (10, 5))
y, a = np.meshgrid(y_points, alt)
x, a = np.meshgrid(x_points, alt)
t, a = np.meshgrid(times, alt)
p = a
p[0, :] = 4
p[1, :] = 16
a = Coord(a, Metadata(standard_name='altitude', units='meters'))
x = Coord(x, Metadata(standard_name='latitude', units='degrees'))
y = Coord(y, Metadata(standard_name='longitude', units='degrees'))
p = Coord(p, Metadata(standard_name='air_pressure', units='Pa'))
t = Coord(t, Metadata(standard_name='time', units=cis_standard_time_unit))
coords = CoordList([x, y, a, p, t])
return UngriddedData(data, Metadata(standard_name='rainfall_flux', long_name="TOTAL RAINFALL RATE: LS+CONV KG/M2/S",
units="kg m-2 s-1", missing_value=-999), coords)
