def test_create_data_object_with_valid_datetime(self): import datetime from cis.time_util import convert_datetime_to_std_time data = self.product().create_data_object([self.filename], True) assert(data.coord('time').data[3] == convert_datetime_to_std_time(datetime.datetime(2012, 8, 25, 15, 32, 0o3))) assert(data.coord('time').data[4] == convert_datetime_to_std_time(datetime.datetime(2012, 8, 26)))
def check_valid_min_max_args(min_val, max_val, step, parser, range_axis): """ If a val range was specified, checks that they are valid numbers and the min is less than the max """ from cis.parse_datetime import parse_as_number_or_datetime, parse_as_float_or_time_delta, \ parse_datetimestr_to_std_time from cis.time_util import convert_datetime_to_std_time import datetime ax_range = {} if min_val is not None: dt = parse_as_number_or_datetime(min_val, range_axis + "min", parser) if isinstance(dt, list): ax_range[range_axis + "min"] = convert_datetime_to_std_time(datetime.datetime(*dt)) else: ax_range[range_axis + "min"] = dt if max_val is not None: dt = parse_as_number_or_datetime(max_val, range_axis + "max", parser) if isinstance(dt, list): ax_range[range_axis + "max"] = convert_datetime_to_std_time(datetime.datetime(*dt)) else: ax_range[range_axis + "max"] = dt if step is not None: ax_range[range_axis + "step"] = parse_as_float_or_time_delta(step, range_axis + "step", parser) return ax_range
def test_can_subset_ungridded_data_by_time(self): data = cis.test.util.mock.make_regular_4d_ungridded_data() time_coord = data.coord('time') constraint = subset_constraint.UngriddedSubsetConstraint() constraint.set_limit(time_coord, time_util.convert_datetime_to_std_time(datetime.datetime(1984, 8, 28)), time_util.convert_datetime_to_std_time(datetime.datetime(1984, 8, 29))) subset = constraint.constrain(data) assert (subset.data.tolist() == [2, 3, 7, 8, 12, 13, 17, 18, 22, 23, 27, 28, 32, 33, 37, 38, 42, 43, 47, 48])
def test_that_can_calculate_mid_point_between_two_datetime(self): from cis.time_util import calculate_mid_time t1 = convert_datetime_to_std_time(dt.datetime(2010, 0o2, 0o5, 0, 0, 0)) t2 = convert_datetime_to_std_time(dt.datetime(2010, 0o2, 0o6, 0, 0, 0)) tm = calculate_mid_time(t1, t2) eq_( tm, convert_datetime_to_std_time(dt.datetime(2010, 0o2, 0o5, 12, 0, 0)))
def test_can_subset_ungridded_data_by_time_altitude(self): data = cis.test.util.mock.make_regular_4d_ungridded_data() time_coord = data.coord('time') alt_coord = data.coord('altitude') constraint = subset_constraint.UngriddedSubsetConstraint() constraint.set_limit(time_coord, time_util.convert_datetime_to_std_time(datetime.datetime(1984, 8, 28)), time_util.convert_datetime_to_std_time(datetime.datetime(1984, 8, 29))) constraint.set_limit(alt_coord, 45.0, 75.0) subset = constraint.constrain(data) assert (subset.data.tolist() == [27, 28, 32, 33, 37, 38])
def __new__(cls, lat=None, lon=None, alt=None, pres=None, t=None, val=None): """ Small constructor for the HyperPoint named tuple to allow optional arguments and set-up value list. """ # If no value was specified create an empty list, otherwise create a list with one entry if val is None or val == []: val = [] else: val = [val] # If t was given as a datetime we need to convert it into our standard time if isinstance(t, datetime.datetime): t = convert_datetime_to_std_time(t) point = super(HyperPoint, cls).__new__(cls, lat, lon, alt, pres, t, val) # Store the coordinate tuple for this point in case we need it later point.coord_tuple = point.get_coord_tuple() return point
def convert_to_std_time(self, time_stamp_info=None): """ Convert this coordinate to standard time. It will use either: the units of the coordinate if it is a cf_units Unit, or the first word of the units, combined with the time stamp (if the timestamp is not given an error is thrown). :param time_stamp_info: the time stamp info from the file, None if it does not exist """ from cis.time_util import convert_time_since_to_std_time, cis_standard_time_unit, \ convert_time_using_time_stamp_info_to_std_time, convert_datetime_to_std_time from cf_units import Unit if isinstance(self.units, Unit): self._data = convert_time_since_to_std_time(self.data, self.units) elif str(self.units).lower().startswith('datetime'): self._data = convert_datetime_to_std_time(self.data) else: if time_stamp_info is None: raise ValueError( "File must have time stamp info if converting without 'since' in units definition" ) self._data = convert_time_using_time_stamp_info_to_std_time( self.data, self.units, time_stamp_info) self.units = cis_standard_time_unit
def modified(self, lat=None, lon=None, alt=None, pres=None, t=None, val=None): """Creates a HyperPoint with modified values. :param lat: :param lon: :param alt: :param pres: :param t: :param val: :return: """ values = [v for v in self] if lat is not None: values[HyperPoint.LATITUDE] = lat if lon is not None: values[HyperPoint.LONGITUDE] = lon if alt is not None: values[HyperPoint.ALTITUDE] = alt if pres is not None: values[HyperPoint.AIR_PRESSURE] = pres if t is not None: if isinstance(t, datetime.datetime): values[HyperPoint.TIME] = convert_datetime_to_std_time(t) if val is not None: if val == []: values[HyperPoint.VAL] = [] else: values[HyperPoint.VAL] = [val] point = super(HyperPoint, self).__new__(HyperPoint, *values) point.coord_tuple = point.get_coord_tuple() return point
def test_convert_julian_date_to_std_time(self): import numpy as np from cis.time_util import convert_datetime_to_std_time julian_days = np.array([ 2454637.8091, 2454637.8092, 2454637.8097, 2454637.8197, 2454638.8097, 2454657.8097, 2454737.8197, 2456638.8097, 2464657.8097 ]) std_days = convert_julian_date_to_std_time(julian_days) ref = convert_datetime_to_std_time([ dt.datetime(2008, 6, 20, 7, 25, 6), dt.datetime(2008, 6, 20, 7, 25, 15), dt.datetime(2008, 6, 20, 7, 25, 58), dt.datetime(2008, 6, 20, 7, 40, 22), dt.datetime(2008, 6, 21, 7, 25, 58), dt.datetime(2008, 7, 10, 7, 25, 58), dt.datetime(2008, 9, 28, 7, 40, 22), dt.datetime(2013, 12, 12, 7, 25, 58), dt.datetime(2035, 11, 26, 7, 25, 58) ]) eq_(julian_days.shape, std_days.shape) assert np.allclose(std_days, ref)
def test_GIVEN_single_point_in_cube_WHEN_iterate_THEN_return_point_in_middle(self): sample_cube = make_square_5x3_2d_cube_with_time(offset=0, time_offset=0) data_point = make_dummy_ungridded_data_single_point(0.5, 0.5, 1.2, time=datetime.datetime(1984, 8, 28, 0, 0)) coord_map = make_coord_map(sample_cube, data_point) coords = sample_cube.coords() for (hpi, ci, shi) in coord_map: coord = coords[ci] if coord.ndim > 1: raise NotImplementedError("Co-location of data onto a cube with a coordinate of dimension greater" " than one is not supported (coordinate %s)", coord.name()) # Ensure that bounds exist. if not coord.has_bounds(): coord.guess_bounds() constraint = BinnedCubeCellOnlyConstraint() data_index.create_indexes(constraint, coords, data_point.get_non_masked_points(), coord_map) iterator = constraint.get_iterator(False, coord_map, coords, data_point.get_non_masked_points(), None, sample_cube, None) final_points_index = [(out_index, hp, points) for out_index, hp, points in iterator] assert_that(len(final_points_index), is_(1), "There is one mapping from sample_cube to the final grid") assert_that(final_points_index[0][0], is_((2, 1, 1)), "The points should map to index") assert_that(final_points_index[0][1], is_(HyperPoint(lat=0, lon=0, t=datetime.datetime(1984, 8, 28))), "The points should map to index") assert_that(final_points_index[0][2].latitudes, is_([0.5]), "The points should map to index") assert_that(final_points_index[0][2].longitudes, is_([0.5]), "The points should map to index") assert_that(final_points_index[0][2].times, is_([convert_datetime_to_std_time(datetime.datetime(1984, 8, 28, 0, 0))]), "The points should map to index") assert_that(final_points_index[0][2].vals, is_([1.2]), "The points should map to index")
def test_change_ug_data_year(self): from cis.test.util.mock import make_regular_2d_with_time_ungridded_data from cis.time_util import convert_datetime_to_std_time from datetime import datetime ug = make_regular_2d_with_time_ungridded_data() change_year_of_ungridded_data(ug, 2007) eq_( ug.coord('time').points[0, 0], convert_datetime_to_std_time(datetime(2007, 8, 27)))
def make_square_5x3_2d_cube_with_scalar_time(time_offset=0): """ Makes a well defined cube of shape 5x3 with data as follows array([[1,2,3], [4,5,6], [7,8,9], [10,11,12], [13,14,15]]) and coordinates in latitude: array([ -10, -5, 0, 5, 10 ]) longitude: array([ -5, 0, 5 ]) time: 1984-08-27 time_bounds: [1984-08-22, 1984-09-01] """ import numpy as np from iris.cube import Cube from iris.coords import DimCoord from cis.time_util import cis_standard_time_unit t0 = datetime.datetime(1984, 8, 27) time_nums = convert_datetime_to_std_time(t0 + datetime.timedelta(days=time_offset)) time = DimCoord(time_nums, standard_name='time', bounds=[convert_datetime_to_std_time(t0 - datetime.timedelta(days=5)), convert_datetime_to_std_time(t0 + datetime.timedelta(days=5))], units=cis_standard_time_unit) latitude = DimCoord(np.arange(-10., 11., 5), var_name='lat', standard_name='latitude', units='degrees') longitude = DimCoord(np.arange(-5., 6., 5), var_name='lon', standard_name='longitude', units='degrees') data = np.reshape(np.arange(15) + 1.0, (5, 3)) cube = Cube(data, dim_coords_and_dims=[(latitude, 0), (longitude, 1)], var_name='dummy') cube.add_aux_coord(time) return cube
def make_square_NxM_2d_cube_with_time(start_lat=-10, end_lat=10, lat_point_count=5, start_lon=-5, end_lon=5, lon_point_count=3, time_offset=0): """ Makes a well defined cube of shape 5x3 with data as follows arr([[[ 1. 2. 3. 4. 5. 6. 7.] [ 8. 9. 10. 11. 12. 13. 14.] [ 15. 16. 17. 18. 19. 20. 21.]] [[ 22. 23. 24. 25. 26. 27. 28.] [ 29. 30. 31. 32. 33. 34. 35.] [ 36. 37. 38. 39. 40. 41. 42.]] [[ 43. 44. 45. 46. 47. 48. 49.] [ 50. 51. 52. 53. 54. 55. 56.] [ 57. 58. 59. 60. 61. 62. 63.]] [[ 64. 65. 66. 67. 68. 69. 70.] [ 71. 72. 73. 74. 75. 76. 77.] [ 78. 79. 80. 81. 82. 83. 84.]] [[ 85. 86. 87. 88. 89. 90. 91.] [ 92. 93. 94. 95. 96. 97. 98.] [ 99. 100. 101. 102. 103. 104. 105.]]]) and coordinates in latitude: array([ -10, -5, 0, 5, 10 ]) longitude: array([ -5, 0, 5 ]) time: array([1984-08-27, 1984-08-28, 1984-08-29, 1984-08-30, 1984-08-31, 1984-09-01, 1984-09-02]) They are different lengths to make it easier to distinguish. Note the latitude increases as you step through the array in order - so downwards as it's written above """ import numpy as np from iris.cube import Cube from iris.coords import DimCoord import datetime t0 = datetime.datetime(1984, 8, 27) times = np.array([t0 + datetime.timedelta(days=d + time_offset) for d in range(7)]) time_nums = convert_datetime_to_std_time(times) time = DimCoord(time_nums, standard_name='time') latitude = DimCoord(np.linspace(start_lat, end_lat, lat_point_count), standard_name='latitude', units='degrees') longitude = DimCoord(np.linspace(start_lon, end_lon, lon_point_count), standard_name='longitude', units='degrees') data = np.reshape(np.arange(lat_point_count * lon_point_count * 7) + 1.0, (lat_point_count, lon_point_count, 7)) cube = Cube(data, dim_coords_and_dims=[(latitude, 0), (longitude, 1), (time, 2)], var_name='dummy') return cube
def test_GIVEN_single_point_in_cube_WHEN_iterate_THEN_return_point_in_middle( self): sample_cube = make_square_5x3_2d_cube_with_time(offset=0, time_offset=0) data_point = make_dummy_ungridded_data_single_point( 0.5, 0.5, 1.2, time=datetime.datetime(1984, 8, 28, 0, 0)) coord_map = make_coord_map(sample_cube, data_point) coords = sample_cube.coords() for (hpi, ci, shi) in coord_map: coord = coords[ci] if coord.ndim > 1: raise NotImplementedError( "Co-location of data onto a cube with a coordinate of dimension greater" " than one is not supported (coordinate %s)", coord.name()) # Ensure that bounds exist. if not coord.has_bounds(): coord.guess_bounds() constraint = BinnedCubeCellOnlyConstraint() data_index.create_indexes(constraint, coords, data_point.get_non_masked_points(), coord_map) iterator = constraint.get_iterator(False, coord_map, coords, data_point.get_non_masked_points(), None, sample_cube, None) final_points_index = [(out_index, hp, points) for out_index, hp, points in iterator] assert_that(len(final_points_index), is_(1), "There is one mapping from sample_cube to the final grid") assert_that(final_points_index[0][0], is_((2, 1, 1)), "The points should map to index") assert_that( final_points_index[0][1], is_(HyperPoint(lat=0, lon=0, t=datetime.datetime(1984, 8, 28))), "The points should map to index") assert_that(final_points_index[0][2].latitudes, is_([0.5]), "The points should map to index") assert_that(final_points_index[0][2].longitudes, is_([0.5]), "The points should map to index") assert_that( final_points_index[0][2].times, is_([ convert_datetime_to_std_time( datetime.datetime(1984, 8, 28, 0, 0)) ]), "The points should map to index") assert_that(final_points_index[0][2].vals, is_([1.2]), "The points should map to index")
def create_coords(self, filenames, variable=None): from cis.data_io.ungridded_data import Metadata from numpy import genfromtxt, NaN from cis.exceptions import InvalidVariableError from cis.time_util import convert_datetime_to_std_time import dateutil.parser as du array_list = [] for filename in filenames: try: array_list.append(genfromtxt(filename, dtype="f8,f8,f8,O,f8", names=['latitude', 'longitude', 'altitude', 'time', 'value'], delimiter=',', missing_values='', usemask=True, invalid_raise=True, converters={"time": du.parse})) except: raise IOError('Unable to read file ' + filename) data_array = utils.concatenate(array_list) n_elements = len(data_array['latitude']) coords = CoordList() coords.append(Coord(data_array["latitude"], Metadata(standard_name="latitude", shape=(n_elements,), units="degrees_north"))) coords.append(Coord(data_array["longitude"], Metadata(standard_name="longitude", shape=(n_elements,), units="degrees_east"))) coords.append( Coord(data_array["altitude"], Metadata(standard_name="altitude", shape=(n_elements,), units="meters"))) time_arr = convert_datetime_to_std_time(data_array["time"]) time = Coord(time_arr, Metadata(standard_name="time", shape=(n_elements,), units="days since 1600-01-01 00:00:00")) coords.append(time) if variable: try: data = UngriddedData(data_array['value'], Metadata(name="value", shape=(n_elements,), units="unknown", missing_value=NaN), coords) except: InvalidVariableError("Value column does not exist in file " + filenames) return data else: return UngriddedCoordinates(coords)
def test_convert_julian_date_to_std_time(self): import numpy as np from cis.time_util import convert_datetime_to_std_time julian_days = np.array([2454637.8091, 2454637.8092, 2454637.8097, 2454637.8197, 2454638.8097, 2454657.8097, 2454737.8197, 2456638.8097, 2464657.8097]) std_days = convert_julian_date_to_std_time(julian_days) ref = convert_datetime_to_std_time([dt.datetime(2008, 6, 20, 7, 25, 6), dt.datetime(2008, 6, 20, 7, 25, 15), dt.datetime(2008, 6, 20, 7, 25, 58), dt.datetime(2008, 6, 20, 7, 40, 22), dt.datetime(2008, 6, 21, 7, 25, 58), dt.datetime(2008, 7, 10, 7, 25, 58), dt.datetime(2008, 9, 28, 7, 40, 22), dt.datetime(2013, 12, 12, 7, 25, 58), dt.datetime(2035, 11, 26, 7, 25, 58)]) eq_(julian_days.shape, std_days.shape) assert np.allclose(std_days, ref)
def convert_to_std_time(self, time_stamp_info=None): """ Convert this coordinate to standard time. It will use either: the units of the coordinate if it is a cf_units Unit, or the first word of the units, combined with the time stamp (if the timestamp is not given an error is thrown). :param time_stamp_info: the time stamp info from the file, None if it does not exist """ from cis.time_util import convert_time_since_to_std_time, cis_standard_time_unit, \ convert_time_using_time_stamp_info_to_std_time, convert_datetime_to_std_time from cf_units import Unit if isinstance(self.units, Unit): self._data = convert_time_since_to_std_time(self.data, self.units) elif str(self.units).lower().startswith('datetime'): self._data = convert_datetime_to_std_time(self.data) else: if time_stamp_info is None: raise ValueError("File must have time stamp info if converting without 'since' in units definition") self._data = convert_time_using_time_stamp_info_to_std_time(self.data, self.units, time_stamp_info) self.units = cis_standard_time_unit
def test_that_can_parse_datetimestr_to_obj(): from cis.time_util import convert_datetime_to_std_time import datetime as dt # when not specifying the hours, minutes or seconds, 0 is used eq_(parse_datetimestr_to_std_time("2010-02-05 02:15:45"), convert_datetime_to_std_time(dt.datetime(2010, 2, 5, 2, 15, 45))) eq_(parse_datetimestr_to_std_time("2010-02-05 02:15"), convert_datetime_to_std_time(dt.datetime(2010, 2, 5, 2, 15, 0))) eq_(parse_datetimestr_to_std_time("2010-02-05 02"), convert_datetime_to_std_time(dt.datetime(2010, 2, 5, 2, 0, 0))) eq_(parse_datetimestr_to_std_time("2010-02-05"), convert_datetime_to_std_time(dt.datetime(2010, 2, 5, 0, 0, 0))) # GOTCHA: when not specifying an element of a date (i.e. the year, month or day), the current date is used now = dt.datetime.now() eq_(parse_datetimestr_to_std_time("2010-02-05"), convert_datetime_to_std_time(dt.datetime(2010, 2, 5))) eq_(parse_datetimestr_to_std_time("2010-12"), convert_datetime_to_std_time(dt.datetime(2010, 12, now.day))) eq_(parse_datetimestr_to_std_time("2010-"), convert_datetime_to_std_time(dt.datetime(2010, now.month, now.day)))
def index_data(self, coords, hyper_points, coord_map): """ Index the data that falls inside the grid cells :param coords: coordinates of grid :param hyper_points: list of HyperPoints to index :param coord_map: list of tuples relating index in HyperPoint to index in coords and in coords to be iterated over """ # create bounds in correct order hp_coords = [] coord_descreasing = [False] * len(coords) coord_lengths = [0] * len(coords) lower_bounds = [None] * len(coords) max_bounds = [None] * len(coords) for (hpi, ci, shi) in coord_map: coord = coords[ci] # Coordinates must be monotonic; determine whether increasing or decreasing. if len(coord.points) > 1: if coord.points[1] < coord.points[0]: coord_descreasing[shi] = True coord_lengths[shi] = len(coord.points) if coord_descreasing[shi]: lower_bounds[shi] = coord.bounds[::-1, 1] max_bounds[shi] = coord.bounds[0, 1] else: lower_bounds[shi] = coord.bounds[::, 0] max_bounds[shi] = coord.bounds[-1, 1] hp_coord = hyper_points.coords[hpi] if isinstance(hp_coord[0], datetime.datetime): hp_coord = convert_datetime_to_std_time(hp_coord) hp_coords.append(hp_coord) bounds_coords_max = list(zip(lower_bounds, hp_coords, max_bounds)) # stack for each coordinate # where the coordinate is larger than the maximum set to -1 # otherwise search in the sorted coordinate to find all the index of the hyperpoints # The choice of 'left' or 'right' and '<' and '<=' determines which # cell is chosen when the coordinate is equal to the boundary. # -1 or M_i indicates the point is outside the grid. # Output is a list of coordinates which lists the indexes where the hyper points # should be located in the grid indices = np.vstack( np.where(ci < max_coordinate_value, np.searchsorted(bi, ci, side='right') - 1, -1) for bi, ci, max_coordinate_value in bounds_coords_max) # D-tuple giving the shape of the output grid grid_shape = tuple(len(bi_ci[0]) for bi_ci in bounds_coords_max) # shape (N,) telling which points actually fall within the grid, # i.e. have indexes that are not -1 and are not masked data points grid_mask = np.all( (indices >= 0) & (ma.getmaskarray(hyper_points.data) == False), axis=0) # if the coordinate was decreasing then correct the indices for this cell for indices_slice, decreasing, coord_length in zip( range(indices.shape[0]), coord_descreasing, coord_lengths): if decreasing: # indices[indices_slice] += (coord_length - 1) - indices[indices_slice] indices[indices_slice] *= -1 indices[indices_slice] += (coord_length - 1) # shape (N,) containing negative scalar cell numbers for each # input point (sequence doesn't matter so long as they are unique), or # -1 for points outside the grid. # # Possibly numpy.lexsort could be used to avoid the need for this, # although we'd have to be careful about points outside the grid. self.cell_numbers = np.where( grid_mask, np.tensordot(np.cumproduct((1, ) + grid_shape[:-1]), indices, axes=1), -1) # Sort everything by cell number self.sort_order = np.argsort(self.cell_numbers) self.cell_numbers = self.cell_numbers[self.sort_order] self._indices = indices[:, self.sort_order] self.hp_coords = [hp_coord[self.sort_order] for hp_coord in hp_coords]
def convert_datetime_to_standard_time(self): from cis.time_util import convert_datetime_to_std_time, cis_standard_time_unit self._data = convert_datetime_to_std_time(self.data) self.units = cis_standard_time_unit
def convert_datetime_to_standard_time(self): from cis.time_util import convert_datetime_to_std_time, cis_standard_time_unit self._data = convert_datetime_to_std_time(self.data) self.units = str(cis_standard_time_unit) self.metadata.calendar = cis_standard_time_unit.calendar
def test_that_can_calculate_mid_point_between_two_datetime(): from cis.time_util import calculate_mid_time t1 = convert_datetime_to_std_time(dt.datetime(2010, 02, 05, 0, 0, 0)) t2 = convert_datetime_to_std_time(dt.datetime(2010, 02, 06, 0, 0, 0)) tm = calculate_mid_time(t1, t2) eq_(tm, convert_datetime_to_std_time(dt.datetime(2010, 02, 05, 12, 0, 0)))
def make_mock_cube(lat_dim_length=5, lon_dim_length=3, lon_range=None, alt_dim_length=0, pres_dim_length=0, time_dim_length=0, horizontal_offset=0, altitude_offset=0, pressure_offset=0, time_offset=0, data_offset=0, surf_pres_offset=0, hybrid_ht_len=0, hybrid_pr_len=0, geopotential_height=False, dim_order=None, mask=False): """ Makes a cube of any shape required, with coordinate offsets from the default available. If no arguments are given get a 5x3 cube of the form: array([[1,2,3], [4,5,6], [7,8,9], [10,11,12], [13,14,15]]) and coordinates in latitude: array([ -10, -5, 0, 5, 10 ]) longitude: array([ -5, 0, 5 ]) :param lat_dim_length: Latitude grid length :param lon_dim_length: Longitude grid length :param alt_dim_length: Altitude grid length :param pres_dim_length: Pressure grid length :param time_dim_length: Time grid length :param horizontal_offset: Offset from the default grid, in degrees, in lat and lon :param altitude_offset: Offset from the default grid in altitude :param pressure_offset: Offset from the default grid in pressure :param time_offset: Offset from the default grid in time :param data_offset: Offset from the default data values :param surf_pres_offset: Offset for the optional surface pressure field :param hybrid_ht_len: Hybrid height grid length :param hybrid_pr_len: Hybrid pressure grid length :param geopotential_height: Include a geopotential height field when calcluting a hybrid pressure? (default False) :param dim_order: List of 'lat', 'lon', 'alt', 'pres', 'time' in the order in which the dimensions occur :param mask: A mask to apply to the data, this should be either a scalar or the same shape as the data :return: A cube with well defined data. """ import iris from iris.aux_factory import HybridHeightFactory, HybridPressureFactory data_size = 1 DIM_NAMES = ['lat', 'lon', 'alt', 'pres', 'time', 'hybrid_ht', 'hybrid_pr'] dim_lengths = [lat_dim_length, lon_dim_length, alt_dim_length, pres_dim_length, time_dim_length, hybrid_ht_len, hybrid_pr_len] lon_range = lon_range or (-5., 5.) if dim_order is None: dim_order = list(DIM_NAMES) if any([True for d in dim_order if d not in DIM_NAMES]): raise ValueError("dim_order contains unrecognised name") for idx, dim in enumerate(DIM_NAMES): if dim_lengths[idx] == 0 and dim in dim_order: del dim_order[dim_order.index(dim)] coord_map = {} for idx, dim in enumerate(dim_order): coord_map[dim] = dim_order.index(dim) coord_list = [None] * len(coord_map) if lat_dim_length: coord_list[coord_map['lat']] = (DimCoord(np.linspace(-10., 10., lat_dim_length) + horizontal_offset, standard_name='latitude', units='degrees', var_name='lat'), coord_map['lat']) data_size *= lat_dim_length if lon_dim_length: coord_list[coord_map['lon']] = ( DimCoord(np.linspace(lon_range[0], lon_range[1], lon_dim_length) + horizontal_offset, standard_name='longitude', units='degrees', var_name='lon'), coord_map['lon']) data_size *= lon_dim_length if alt_dim_length: coord_list[coord_map['alt']] = (DimCoord(np.linspace(0., 7., alt_dim_length) + altitude_offset, standard_name='altitude', units='metres', var_name='alt'), coord_map['alt']) data_size *= alt_dim_length if pres_dim_length: coord_list[coord_map['pres']] = (DimCoord(np.linspace(0., 7., pres_dim_length) + pressure_offset, standard_name='air_pressure', units='hPa', var_name='pres'), coord_map['pres']) data_size *= pres_dim_length if time_dim_length: t0 = datetime.datetime(1984, 8, 27) times = np.array([t0 + datetime.timedelta(days=d + time_offset) for d in range(time_dim_length)]) time_nums = convert_datetime_to_std_time(times) time_bounds = None if time_dim_length == 1: time_bounds = convert_datetime_to_std_time(np.array([times[0] - datetime.timedelta(days=0.5), times[0] + datetime.timedelta(days=0.5)])) coord_list[coord_map['time']] = (DimCoord(time_nums, standard_name='time', units='days since 1600-01-01 00:00:00', var_name='time', bounds=time_bounds), coord_map['time']) data_size *= time_dim_length if hybrid_ht_len: coord_list[coord_map['hybrid_ht']] = (DimCoord(np.arange(hybrid_ht_len, dtype='i8') + 10, "model_level_number", units="1"), coord_map['hybrid_ht']) data_size *= hybrid_ht_len if hybrid_pr_len: coord_list[coord_map['hybrid_pr']] = (DimCoord(np.arange(hybrid_pr_len, dtype='i8'), "atmosphere_hybrid_sigma_pressure_coordinate", units="1"), coord_map['hybrid_pr']) data_size *= hybrid_pr_len data = np.reshape(np.arange(data_size) + data_offset + 1., tuple(len(i[0].points) for i in coord_list)) if mask: data = np.ma.asarray(data) data.mask = mask return_cube = Cube(data, dim_coords_and_dims=coord_list, var_name='rain', standard_name='rainfall_rate', long_name="TOTAL RAINFALL RATE: LS+CONV KG/M2/S", units="kg m-2 s-1") if hybrid_ht_len: return_cube.add_aux_coord(iris.coords.AuxCoord(np.arange(hybrid_ht_len, dtype='i8') + 40, long_name="level_height", units="m", var_name='hybrid_ht'), coord_map['hybrid_ht']) return_cube.add_aux_coord(iris.coords.AuxCoord(np.arange(hybrid_ht_len, dtype='i8') + 50, long_name="sigma", units="1", var_name='sigma'), coord_map['hybrid_ht']) return_cube.add_aux_coord(iris.coords.AuxCoord( np.arange(lat_dim_length * lon_dim_length, dtype='i8').reshape(lat_dim_length, lon_dim_length) + 100, long_name="surface_altitude", units="m"), [coord_map['lat'], coord_map['lon']]) return_cube.add_aux_factory(HybridHeightFactory( delta=return_cube.coord("level_height"), sigma=return_cube.coord("sigma"), orography=return_cube.coord("surface_altitude"))) elif hybrid_pr_len: return_cube.add_aux_coord(iris.coords.AuxCoord(np.arange(hybrid_pr_len, dtype='i8') + 40, long_name="hybrid A coefficient at layer midpoints", units="Pa", var_name='a'), coord_map['hybrid_pr']) return_cube.add_aux_coord(iris.coords.AuxCoord(np.arange(hybrid_pr_len, dtype='f8') + 50, long_name="hybrid B coefficient at layer midpoints", units="1", var_name='b'), coord_map['hybrid_pr']) return_cube.add_aux_coord( iris.coords.AuxCoord(np.arange(lat_dim_length * lon_dim_length * time_dim_length, dtype='i8') .reshape(lat_dim_length, lon_dim_length, time_dim_length) * 100000 + surf_pres_offset, "surface_air_pressure", units="Pa"), [coord_map['lat'], coord_map['lon'], coord_map['time']]) if geopotential_height: return_cube.add_aux_coord(iris.coords.AuxCoord( np.arange(lat_dim_length * lon_dim_length * time_dim_length * hybrid_pr_len, dtype='i8') .reshape(lat_dim_length, lon_dim_length, time_dim_length, hybrid_pr_len) + 10, "altitude", long_name="Geopotential height at layer midpoints", units="meter"), [coord_map['lat'], coord_map['lon'], coord_map['time'], coord_map['hybrid_pr']]) return_cube.add_aux_factory(HybridPressureFactory( delta=return_cube.coord("hybrid A coefficient at layer midpoints"), sigma=return_cube.coord("hybrid B coefficient at layer midpoints"), surface_air_pressure=return_cube.coord("surface_air_pressure"))) for coord in return_cube.coords(dim_coords=True): if coord.bounds is None: coord.guess_bounds() return return_cube
def index_data(self, coords, hyper_points, coord_map): """ Index the data that falls inside the grid cells :param coords: coordinates of grid :param hyper_points: list of HyperPoints to index :param coord_map: list of tuples relating index in HyperPoint to index in coords and in coords to be iterated over """ # create bounds in correct order hp_coords = [] coord_descreasing = [False] * len(coords) coord_lengths = [0] * len(coords) lower_bounds = [None] * len(coords) max_bounds = [None] * len(coords) for (hpi, ci, shi) in coord_map: coord = coords[ci] # Coordinates must be monotonic; determine whether increasing or decreasing. if len(coord.points) > 1: if coord.points[1] < coord.points[0]: coord_descreasing[shi] = True coord_lengths[shi] = len(coord.points) if coord_descreasing[shi]: lower_bounds[shi] = coord.bounds[::-1, 1] max_bounds[shi] = coord.bounds[0, 1] else: lower_bounds[shi] = coord.bounds[::, 0] max_bounds[shi] = coord.bounds[-1, 1] hp_coord = hyper_points.coords[hpi] if isinstance(hp_coord[0], datetime.datetime): hp_coord = convert_datetime_to_std_time(hp_coord) hp_coords.append(hp_coord) bounds_coords_max = list(zip(lower_bounds, hp_coords, max_bounds)) # stack for each coordinate # where the coordinate is larger than the maximum set to -1 # otherwise search in the sorted coordinate to find all the index of the hyperpoints # The choice of 'left' or 'right' and '<' and '<=' determines which # cell is chosen when the coordinate is equal to the boundary. # -1 or M_i indicates the point is outside the grid. # Output is a list of coordinates which lists the indexes where the hyper points # should be located in the grid indices = np.vstack( np.where( ci < max_coordinate_value, np.searchsorted(bi, ci, side='right') - 1, -1) for bi, ci, max_coordinate_value in bounds_coords_max) # D-tuple giving the shape of the output grid grid_shape = tuple(len(bi_ci[0]) for bi_ci in bounds_coords_max) # shape (N,) telling which points actually fall within the grid, # i.e. have indexes that are not -1 and are not masked data points grid_mask = np.all( (indices >= 0) & (ma.getmaskarray(hyper_points.data) == False), axis=0) # if the coordinate was decreasing then correct the indices for this cell for indices_slice, decreasing, coord_length in zip(range(indices.shape[0]), coord_descreasing, coord_lengths): if decreasing: # indices[indices_slice] += (coord_length - 1) - indices[indices_slice] indices[indices_slice] *= -1 indices[indices_slice] += (coord_length - 1) # shape (N,) containing negative scalar cell numbers for each # input point (sequence doesn't matter so long as they are unique), or # -1 for points outside the grid. # # Possibly numpy.lexsort could be used to avoid the need for this, # although we'd have to be careful about points outside the grid. self.cell_numbers = np.where( grid_mask, np.tensordot( np.cumproduct((1,) + grid_shape[:-1]), indices, axes=1 ), -1) # Sort everything by cell number self.sort_order = np.argsort(self.cell_numbers) self.cell_numbers = self.cell_numbers[self.sort_order] self._indices = indices[:, self.sort_order] self.hp_coords = [hp_coord[self.sort_order] for hp_coord in hp_coords]
def make_regular_4d_ungridded_data(): """ Makes a well defined ungridded data object of shape 10x5 with data as follows data: [[ 1. 2. 3. 4. 5.] [ 6. 7. 8. 9. 10.] [ 11. 12. 13. 14. 15.] [ 16. 17. 18. 19. 20.] [ 21. 22. 23. 24. 25.] [ 26. 27. 28. 29. 30.] [ 31. 32. 33. 34. 35.] [ 36. 37. 38. 39. 40.] [ 41. 42. 43. 44. 45.] [ 46. 47. 48. 49. 50.]] latitude: [[-10. -5. 0. 5. 10.] [-10. -5. 0. 5. 10.] [-10. -5. 0. 5. 10.] [-10. -5. 0. 5. 10.] [-10. -5. 0. 5. 10.] [-10. -5. 0. 5. 10.] [-10. -5. 0. 5. 10.] [-10. -5. 0. 5. 10.] [-10. -5. 0. 5. 10.] [-10. -5. 0. 5. 10.]] longitude: [[-5. -2.5 0. 2.5 5. ] [-5. -2.5 0. 2.5 5. ] [-5. -2.5 0. 2.5 5. ] [-5. -2.5 0. 2.5 5. ] [-5. -2.5 0. 2.5 5. ] [-5. -2.5 0. 2.5 5. ] [-5. -2.5 0. 2.5 5. ] [-5. -2.5 0. 2.5 5. ] [-5. -2.5 0. 2.5 5. ] [-5. -2.5 0. 2.5 5. ]] altitude: [[ 0. 0. 0. 0. 0.] [ 10. 10. 10. 10. 10.] [ 20. 20. 20. 20. 20.] [ 30. 30. 30. 30. 30.] [ 40. 40. 40. 40. 40.] [ 50. 50. 50. 50. 50.] [ 60. 60. 60. 60. 60.] [ 70. 70. 70. 70. 70.] [ 80. 80. 80. 80. 80.] [ 90. 90. 90. 90. 90.]] pressure: [[ 4. 4. 4. 4. 4.] [ 16. 16. 16. 16. 16.] [ 20. 20. 20. 20. 20.] [ 30. 30. 30. 30. 30.] [ 40. 40. 40. 40. 40.] [ 50. 50. 50. 50. 50.] [ 60. 60. 60. 60. 60.] [ 70. 70. 70. 70. 70.] [ 80. 80. 80. 80. 80.] [ 90. 90. 90. 90. 90.]] time: [[1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31] [1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31] [1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31] [1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31] [1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31] [1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31] [1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31] [1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31] [1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31] [1984-08-27 1984-08-28 1984-08-29 1984-08-30 1984-08-31]] They are shaped to represent a typical lidar type satelite data set. """ import numpy as np from cis.data_io.Coord import CoordList, Coord from cis.data_io.ungridded_data import UngriddedData, Metadata from cis.time_util import cis_standard_time_unit import datetime x_points = np.linspace(-10, 10, 5) y_points = np.linspace(-5, 5, 5) t0 = datetime.datetime(1984, 8, 27) times = convert_datetime_to_std_time(np.array([t0 + datetime.timedelta(days=d) for d in range(5)])) alt = np.linspace(0, 90, 10) data = np.reshape(np.arange(50) + 1.0, (10, 5)) y, a = np.meshgrid(y_points, alt) x, a = np.meshgrid(x_points, alt) t, a = np.meshgrid(times, alt) p = a p[0, :] = 4 p[1, :] = 16 a = Coord(a, Metadata(standard_name='altitude', units='meters')) x = Coord(x, Metadata(standard_name='latitude', units='degrees')) y = Coord(y, Metadata(standard_name='longitude', units='degrees')) p = Coord(p, Metadata(standard_name='air_pressure', units='Pa')) t = Coord(t, Metadata(standard_name='time', units=cis_standard_time_unit)) coords = CoordList([x, y, a, p, t]) return UngriddedData(data, Metadata(standard_name='rainfall_flux', long_name="TOTAL RAINFALL RATE: LS+CONV KG/M2/S", units="kg m-2 s-1", missing_value=-999), coords)