def test_grib1_hybrid_height(self): gm = gribapi.grib_new_from_samples('regular_gg_ml_grib1') gw = GribWrapper(gm) results = grib1_convert(gw) factory, = results[0] self.assertEqual(factory.factory_class, iris.aux_factory.HybridPressureFactory) delta, sigma, ref = factory.args self.assertEqual(delta, {'long_name': 'level_pressure'}) self.assertEqual(sigma, {'long_name': 'sigma'}) self.assertEqual(ref, Reference(name='surface_pressure')) ml_ref = iris.coords.CoordDefn('model_level_number', None, None, cf_units.Unit('1'), {'positive': 'up'}, None, False) lp_ref = iris.coords.CoordDefn(None, 'level_pressure', None, cf_units.Unit('Pa'), {}, None, False) s_ref = iris.coords.CoordDefn(None, 'sigma', None, cf_units.Unit('1'), {}, None, False) aux_coord_defns = [coord._as_defn() for coord, dim in results[8]] self.assertIn(ml_ref, aux_coord_defns) self.assertIn(lp_ref, aux_coord_defns) self.assertIn(s_ref, aux_coord_defns)
def test_grib1_hybrid_height(self): gm = gribapi.grib_new_from_samples('regular_gg_ml_grib1') gw = GribWrapper(gm) results = grib1_convert(gw) factory, = results[0] self.assertEqual(factory.factory_class, HybridPressureFactory) delta, sigma, ref = factory.args self.assertEqual(delta, {'long_name': 'level_pressure'}) self.assertEqual(sigma, {'long_name': 'sigma'}) self.assertEqual(ref, Reference(name='surface_pressure')) coords_and_dims = results[8] coord, = [ co for co, _ in coords_and_dims if co.name() == 'model_level_number' ] self.assertEqual(coord.units, '1') self.assertEqual(coord.attributes['positive'], 'up') coord, = [ co for co, _ in coords_and_dims if co.name() == 'level_pressure' ] self.assertEqual(coord.units, 'Pa') coord, = [co for co, _ in coords_and_dims if co.name() == 'sigma'] self.assertEqual(coord.units, '1')
def test_grib1_hybrid_height(self): gm = gribapi.grib_new_from_samples('regular_gg_ml_grib1') gw = GribWrapper(gm) results = grib1_convert(gw) factory, = results[0] self.assertEqual(factory.factory_class, iris.aux_factory.HybridPressureFactory) delta, sigma, ref = factory.args self.assertEqual(delta, {'long_name': 'level_pressure'}) self.assertEqual(sigma, {'long_name': 'sigma'}) self.assertEqual(ref, Reference(name='surface_pressure')) ml_ref = iris.coords.CoordDefn('model_level_number', None, None, cf_units.Unit('1'), {'positive': 'up'}, None) lp_ref = iris.coords.CoordDefn(None, 'level_pressure', None, cf_units.Unit('Pa'), {}, None) s_ref = iris.coords.CoordDefn(None, 'sigma', None, cf_units.Unit('1'), {}, None) aux_coord_defns = [coord._as_defn() for coord, dim in results[8]] self.assertIn(ml_ref, aux_coord_defns) self.assertIn(lp_ref, aux_coord_defns) self.assertIn(s_ref, aux_coord_defns)
def check(self, dtype): data = np.random.random(1920 * 2560).astype(dtype) cube = iris.cube.Cube(data, standard_name='geopotential_height', units='km') grib_message = gribapi.grib_new_from_samples("GRIB2") data_section(cube, grib_message) gribapi.grib_release(grib_message)
def test_load_probability_forecast(self): # Test GribWrapper interpretation of PDT 4.9 data. # NOTE: # Currently Iris has only partial support for PDT 4.9. # Though it can load the data, key metadata (thresholds) is lost. # At present, we are not testing for this. # Make a testing grib message in memory, with gribapi. grib_message = gribapi.grib_new_from_samples('GRIB2') gribapi.grib_set_long(grib_message, 'productDefinitionTemplateNumber', 9) gribapi.grib_set_string(grib_message, 'stepRange', '10-55') grib_wrapper = iris.fileformats.grib.GribWrapper(grib_message) # Define two expected datetimes for _periodEndDateTime as # gribapi v1.9.16 mis-calculates this. # See https://software.ecmwf.int/wiki/display/GRIB/\ # GRIB+API+version+1.9.18+released try: # gribapi v1.9.16 has no __version__ attribute. gribapi_ver = gribapi.__version__ except AttributeError: gribapi_ver = gribapi.grib_get_api_version() if StrictVersion(gribapi_ver) < StrictVersion('1.9.18'): exp_end_date = datetime.datetime(year=2007, month=3, day=25, hour=12, minute=0, second=0) else: exp_end_date = datetime.datetime(year=2007, month=3, day=25, hour=19, minute=0, second=0) # Check that it captures the statistics time period info. # (And for now, nothing else) self.assertEqual( grib_wrapper._referenceDateTime, datetime.datetime(year=2007, month=3, day=23, hour=12, minute=0, second=0)) self.assertEqual( grib_wrapper._periodStartDateTime, datetime.datetime(year=2007, month=3, day=23, hour=22, minute=0, second=0)) self.assertEqual(grib_wrapper._periodEndDateTime, exp_end_date)
def system_test_grib_patch(self): import gribapi gm = gribapi.grib_new_from_samples("GRIB2") result = gribapi.grib_get_double(gm, "missingValue") new_missing_value = 123456.0 gribapi.grib_set_double(gm, "missingValue", new_missing_value) new_result = gribapi.grib_get_double(gm, "missingValue") self.assertEqual(new_result, new_missing_value)
def test_bounded_altitude_feet(self): cube = iris.cube.Cube([0]) cube.add_aux_coord(iris.coords.AuxCoord( 1500.0, long_name='altitude', units='ft', bounds=np.array([1000.0, 2000.0]))) grib = gribapi.grib_new_from_samples("GRIB2") grib_save_rules.non_hybrid_surfaces(cube, grib) self.assertEqual( gribapi.grib_get_double(grib, "scaledValueOfFirstFixedSurface"), 304.0) self.assertEqual( gribapi.grib_get_double(grib, "scaledValueOfSecondFixedSurface"), 609.0)
def __init__(self, grib_file=None, clone=None, sample=None, gribindex=None): """ Open a message and inform the GRIB file that it's been incremented. If ``grib_file`` is not supplied, the message is cloned from ``GribMessage`` ``clone``. If neither is supplied, the ``GribMessage`` is cloned from ``sample``. If ``index`` is suppliea as a GribIndex, the message is taken from the index. """ #: Unique GRIB ID, for GRIB API interface self.gid = None #: File containing message self.grib_file = None #: GribIndex referencing message self.grib_index = None # Strangely, if I test any of the input variables in an if-clause I # the GRIB API no longer increments the file, so I've enclosed the gid # assignments in try blocks. I wish there were a better way of doing # this. try: self.gid = gribapi.grib_new_from_file(grib_file.file_handle) self.grib_file = grib_file self.grib_file.message += 1 self.grib_file.open_messages.append(self) except AttributeError: pass try: self.gid = gribapi.grib_clone(clone.gid) except AttributeError: pass try: self.gid = gribapi.grib_new_from_samples(sample) except AssertionError: pass try: self.gid = gribapi.grib_new_from_index(gribindex.iid) if not self.gid: raise IndexNotSelectedError("All keys must have selected " "values before receiving message " "from index.") self.grib_index = gribindex gribindex.open_messages.append(self) except AttributeError: pass if not self.gid: raise RuntimeError("Either grib_file, clone, sample or gribindex " "must be provided.")
def test_bounded_altitude_feet(self): cube = iris.cube.Cube([0]) cube.add_aux_coord( iris.coords.AuxCoord(1500.0, long_name='altitude', units='ft', bounds=np.array([1000.0, 2000.0]))) grib = gribapi.grib_new_from_samples("GRIB2") grib_save_rules.non_hybrid_surfaces(cube, grib) self.assertEqual( gribapi.grib_get_double(grib, "scaledValueOfFirstFixedSurface"), 304.0) self.assertEqual( gribapi.grib_get_double(grib, "scaledValueOfSecondFixedSurface"), 609.0)
def test_load_probability_forecast(self): # Test GribWrapper interpretation of PDT 4.9 data. # NOTE: # Currently iris-grib has only partial support for PDT 4.9. # Though it can load the data, key metadata (thresholds) is lost. # At present, we are not testing for this. # Make a testing grib message in memory, with gribapi. grib_message = gribapi.grib_new_from_samples('GRIB2') gribapi.grib_set_long(grib_message, 'productDefinitionTemplateNumber', 9) gribapi.grib_set_string(grib_message, 'stepRange', '10-55') grib_wrapper = iris.fileformats.grib.GribWrapper(grib_message) # Define two expected datetimes for _periodEndDateTime as # gribapi v1.9.16 mis-calculates this. # See https://software.ecmwf.int/wiki/display/GRIB/\ # GRIB+API+version+1.9.18+released try: # gribapi v1.9.16 has no __version__ attribute. gribapi_ver = gribapi.__version__ except AttributeError: gribapi_ver = gribapi.grib_get_api_version() if StrictVersion(gribapi_ver) < StrictVersion('1.9.18'): exp_end_date = datetime.datetime(year=2007, month=3, day=25, hour=12, minute=0, second=0) else: exp_end_date = datetime.datetime(year=2007, month=3, day=25, hour=19, minute=0, second=0) # Check that it captures the statistics time period info. # (And for now, nothing else) self.assertEqual( grib_wrapper._referenceDateTime, datetime.datetime(year=2007, month=3, day=23, hour=12, minute=0, second=0) ) self.assertEqual( grib_wrapper._periodStartDateTime, datetime.datetime(year=2007, month=3, day=23, hour=22, minute=0, second=0) ) self.assertEqual(grib_wrapper._periodEndDateTime, exp_end_date)
def test_bounded_altitude_feet(self): cube = iris.cube.Cube([0]) cube.add_aux_coord( iris.coords.AuxCoord(1500.0, long_name='altitude', units='ft', bounds=np.array([1000.0, 2000.0]))) grib = gribapi.grib_new_from_samples("GRIB2") set_fixed_surfaces(cube, grib) self.assertEqual( gribapi.grib_get_double(grib, "scaledValueOfFirstFixedSurface"), 305.0) # precise ~304.8 self.assertEqual( gribapi.grib_get_double(grib, "scaledValueOfSecondFixedSurface"), 610.0) # precise ~609.6 self.assertEqual( gribapi.grib_get_long(grib, "typeOfFirstFixedSurface"), 102) self.assertEqual( gribapi.grib_get_long(grib, "typeOfSecondFixedSurface"), 102)
def test_warn_unknown_pdts(self): # Test loading of an unrecognised GRIB Product Definition Template. # Get a temporary file by name (deleted afterward by context). with self.temp_filename() as temp_gribfile_path: # Write a test grib message to the temporary file. with open(temp_gribfile_path, 'wb') as temp_gribfile: grib_message = gribapi.grib_new_from_samples('GRIB2') # Set the PDT to something unexpected. gribapi.grib_set_long(grib_message, 'productDefinitionTemplateNumber', 5) gribapi.grib_write(grib_message, temp_gribfile) # Load the message from the file as a cube. cube_generator = iris_grib.load_cubes(temp_gribfile_path) with self.assertRaises(iris.exceptions.TranslationError) as te: cube = next(cube_generator) self.assertEqual( 'Product definition template [5]' ' is not supported', str(te.exception))
def test_theta_level(self): cube = iris.cube.Cube([0]) cube.add_aux_coord( iris.coords.AuxCoord(230.0, standard_name='air_potential_temperature', units='K', attributes={'positive': 'up'}, bounds=np.array([220.0, 240.0]))) grib = gribapi.grib_new_from_samples("GRIB2") set_fixed_surfaces(cube, grib) self.assertEqual( gribapi.grib_get_double(grib, "scaledValueOfFirstFixedSurface"), 220.0) self.assertEqual( gribapi.grib_get_double(grib, "scaledValueOfSecondFixedSurface"), 240.0) self.assertEqual( gribapi.grib_get_long(grib, "typeOfFirstFixedSurface"), 107) self.assertEqual( gribapi.grib_get_long(grib, "typeOfSecondFixedSurface"), 107)
def test_grib1_hybrid_height(self): gm = gribapi.grib_new_from_samples("regular_gg_ml_grib1") gw = GribWrapper(gm) results = convert(gw) factory, = results[0] self.assertEqual(factory.factory_class, iris.aux_factory.HybridPressureFactory) delta, sigma, ref = factory.args self.assertEqual(delta, {"long_name": "level_pressure"}) self.assertEqual(sigma, {"long_name": "sigma"}) self.assertEqual(ref, Reference(name="surface_pressure")) ml_ref = iris.coords.CoordDefn("model_level_number", None, None, iris.unit.Unit("1"), {"positive": "up"}, None) lp_ref = iris.coords.CoordDefn(None, "level_pressure", None, iris.unit.Unit("Pa"), {}, None) s_ref = iris.coords.CoordDefn(None, "sigma", None, iris.unit.Unit("1"), {}, None) aux_coord_defns = [coord._as_defn() for coord, dim in results[8]] self.assertIn(ml_ref, aux_coord_defns) self.assertIn(lp_ref, aux_coord_defns) self.assertIn(s_ref, aux_coord_defns)
def test_theta_level(self): cube = iris.cube.Cube([0]) cube.add_aux_coord(iris.coords.AuxCoord( 230.0, standard_name='air_potential_temperature', units='K', attributes={'positive': 'up'}, bounds=np.array([220.0, 240.0]))) grib = gribapi.grib_new_from_samples("GRIB2") set_fixed_surfaces(cube, grib) self.assertEqual( gribapi.grib_get_double(grib, "scaledValueOfFirstFixedSurface"), 220.0) self.assertEqual( gribapi.grib_get_double(grib, "scaledValueOfSecondFixedSurface"), 240.0) self.assertEqual( gribapi.grib_get_long(grib, "typeOfFirstFixedSurface"), 107) self.assertEqual( gribapi.grib_get_long(grib, "typeOfSecondFixedSurface"), 107)
def test_warn_unknown_pdts(self): # Test loading of an unrecognised GRIB Product Definition Template. # Get a temporary file by name (deleted afterward by context). with self.temp_filename() as temp_gribfile_path: # Write a test grib message to the temporary file. with open(temp_gribfile_path, 'wb') as temp_gribfile: grib_message = gribapi.grib_new_from_samples('GRIB2') # Set the PDT to something unexpected. gribapi.grib_set_long( grib_message, 'productDefinitionTemplateNumber', 5) gribapi.grib_write(grib_message, temp_gribfile) # Load the message from the file as a cube. cube_generator = iris.fileformats.grib.load_cubes( temp_gribfile_path) with self.assertRaises(iris.exceptions.TranslationError) as te: cube = next(cube_generator) self.assertEqual('Product definition template [5]' ' is not supported', str(te.exception))
def test_depth(self): cube = iris.cube.Cube([0]) cube.add_aux_coord( iris.coords.AuxCoord(1, long_name='depth', units='m', bounds=np.array([0., 2]), attributes={'positive': 'down'})) grib = gribapi.grib_new_from_samples("GRIB2") set_fixed_surfaces(cube, grib) self.assertEqual( gribapi.grib_get_double(grib, "scaledValueOfFirstFixedSurface"), 0.) self.assertEqual( gribapi.grib_get_double(grib, "scaledValueOfSecondFixedSurface"), 2) self.assertEqual( gribapi.grib_get_long(grib, "typeOfFirstFixedSurface"), 106) self.assertEqual( gribapi.grib_get_long(grib, "typeOfSecondFixedSurface"), 106)
def test_load_probability_forecast(self): # Test GribWrapper interpretation of PDT 4.9 data. # NOTE: # Currently Iris has only partial support for PDT 4.9. # Though it can load the data, key metadata (thresholds) is lost. # At present, we are not testing for this. # Make a testing grib message in memory, with gribapi. grib_message = gribapi.grib_new_from_samples('GRIB2') gribapi.grib_set_long(grib_message, 'productDefinitionTemplateNumber', 9) gribapi.grib_set_string(grib_message, 'stepRange', '10-55') grib_wrapper = iris.fileformats.grib.GribWrapper(grib_message) # Check that it captures the statistics time period info. # (And for now, nothing else) self.assertEqual( grib_wrapper._referenceDateTime, datetime.datetime(year=2007, month=03, day=23, hour=12, minute=0, second=0) )
def save_pairs_from_cube(cube): """ Convert one or more cubes to (2D cube, GRIB message) pairs. Returns an iterable of tuples each consisting of one 2D cube and one GRIB message ID, the result of the 2D cube being processed by the GRIB save rules. Args: * cube - A :class:`iris.cube.Cube`, :class:`iris.cube.CubeList` or list of cubes. """ x_coords = cube.coords(axis='x', dim_coords=True) y_coords = cube.coords(axis='y', dim_coords=True) if len(x_coords) != 1 or len(y_coords) != 1: raise TranslationError("Did not find one (and only one) x or y coord") # Save each latlon slice2D in the cube for slice2D in cube.slices([y_coords[0], x_coords[0]]): grib_message = gribapi.grib_new_from_samples("GRIB2") _save_rules.run(slice2D, grib_message) yield (slice2D, grib_message)
def as_pairs(cube): """ Convert one or more cubes to (2D cube, GRIB message) pairs. Returns an iterable of tuples each consisting of one 2D cube and one GRIB message ID, the result of the 2D cube being processed by the GRIB save rules. Args: * cube - A :class:`iris.cube.Cube`, :class:`iris.cube.CubeList` or list of cubes. """ x_coords = cube.coords(axis='x', dim_coords=True) y_coords = cube.coords(axis='y', dim_coords=True) if len(x_coords) != 1 or len(y_coords) != 1: raise TranslationError("Did not find one (and only one) x or y coord") # Save each latlon slice2D in the cube for slice2D in cube.slices([y_coords[0], x_coords[0]]): grib_message = gribapi.grib_new_from_samples("GRIB2") _save_rules.run(slice2D, grib_message) yield (slice2D, grib_message)
def test_warn_unknown_pdts(self): # Test loading of an unrecognised GRIB Product Definition Template. # Get a temporary file by name (deleted afterward by context). with self.temp_filename() as temp_gribfile_path: # Write a test grib message to the temporary file. with open(temp_gribfile_path, 'wb') as temp_gribfile: grib_message = gribapi.grib_new_from_samples('GRIB2') # Set the PDT to something unexpected. gribapi.grib_set_long(grib_message, 'productDefinitionTemplateNumber', 5) gribapi.grib_write(grib_message, temp_gribfile) # Load the message from the file as a cube. cube_generator = iris.fileformats.grib.load_cubes( temp_gribfile_path) cube = next(cube_generator) # Check the cube has an extra "warning" attribute. self.assertEqual( cube.attributes['GRIB_LOAD_WARNING'], 'unsupported GRIB2 ProductDefinitionTemplate: #4.5')
def test_warn_unknown_pdts(self): # Test loading of an unrecognised GRIB Product Definition Template. # Get a temporary file by name (deleted afterward by context). with self.temp_filename() as temp_gribfile_path: # Write a test grib message to the temporary file. with open(temp_gribfile_path, 'wb') as temp_gribfile: grib_message = gribapi.grib_new_from_samples('GRIB2') # Set the PDT to something unexpected. gribapi.grib_set_long( grib_message, 'productDefinitionTemplateNumber', 5) gribapi.grib_write(grib_message, temp_gribfile) # Load the message from the file as a cube. cube_generator = iris.fileformats.grib.load_cubes( temp_gribfile_path) cube = cube_generator.next() # Check the cube has an extra "warning" attribute. self.assertEqual( cube.attributes['GRIB_LOAD_WARNING'], 'unsupported GRIB2 ProductDefinitionTemplate: #4.5' )
def __init__(self, grib_file=None, clone=None, sample=None, gribindex=None): """ Open a message and inform the GRIB file that it's been incremented. If ``grib_file`` is not supplied, the message is cloned from ``GribMessage`` ``clone``. If neither is supplied, the ``GribMessage`` is cloned from ``sample``. If ``index`` is supplied as a GribIndex, the message is taken from the index. """ #: Unique GRIB ID, for GRIB API interface self.gid = None #: File containing message self.grib_file = None #: GribIndex referencing message self.grib_index = None if grib_file is not None: self.gid = gribapi.grib_new_from_file(grib_file.file_handle) if self.gid is None: raise IOError("Grib file %s is exhausted" % grib_file.name) self.grib_file = grib_file self.grib_file.message += 1 self.grib_file.open_messages.append(self) elif clone is not None: self.gid = gribapi.grib_clone(clone.gid) elif sample is not None: self.gid = gribapi.grib_new_from_samples(sample) elif gribindex is not None: self.gid = gribapi.grib_new_from_index(gribindex.iid) if not self.gid: raise IndexNotSelectedError("All keys must have selected " "values before receiving message " "from index.") self.grib_index = gribindex gribindex.open_messages.append(self) else: raise RuntimeError("Either grib_file, clone, sample or gribindex " "must be provided.")
def save(source, target, append=False, sample_file=_sample_file): """ Takes a dataset (source) and writes its contents as grib 1 to file-like target. Grib 1 is used (instead of grib 2) because some older forecast visualization software can't read grib 2. This is a heavily modified but none-the-less derivative of the grib saving functions from the iris package. Parameters ---------- source : Dataset A netcdf-like file holding the dataset we want to write as grib. This must contain time, longitude and latitude coordinates in order to infer the grib grid and time params target : string path or file-like Where the contents should be written. If target is a string the file is created or appended to. append : boolean When creating a new file from string you can optionally append to the file. """ if not _has_gribapi: raise ImportError("gripapi is required to write grib files.") if isinstance(target, basestring): grib_file = open(target, "ab" if append else "wb") elif hasattr(target, "write"): if hasattr(target, "mode") and "b" not in target.mode: raise ValueError("Target not binary") grib_file = target else: raise ValueError("Can only save grib to filename or writable") if not 'latitude' in source.variables or not 'longitude' in source.variables: raise ValueError("Did not find either latitude or longitude.") if source['latitude'].ndim != 1 or source['longitude'].ndim != 1: raise ValueError("Latitude and Longitude should be regular.") if not 'time' in source.variables: raise ValueError("Expected time coordinate") # sort the lats and lons source = source.indexed(latitude=np.argsort(source['latitude'].values)) lons = source['longitude'].values if np.any(np.abs(np.diff(lons)) > 180.): # the latitudes must cross the dateline since we only allow 180 # degree wide bounding boxes, and there is more than a 180 degree # difference between longitudes. Instead we try converting to # 0 to 360 degree longitudes before sorting. lons = np.mod(lons, 360) if np.any(np.abs(np.diff(lons)) > 180.): # TODO: I'm sure theres a way to deal with arbitrary longitude # specifications for global data ... but its not a high priority # so that will wait for later. raise ValueError("Longitudes span more than 180 degrees and the dateline?") source['longitude'].values[:] = lons source = source.indexed(longitude=np.argsort(lons)) # iterate over variables, unless they are considered # auxiliary variables (ie, variables used by slocum # but not in grib files). auxilary_variables = ['wind_speed', 'wind_from_direction'] for single_var in (v for k, v in source.noncoordinates.iteritems() if not k in auxilary_variables): # then iterate over time slices iter_time = (single_var.indexed(**{'time': [i]}) for i in range(single_var.coordinates['time'].size)) for obj in iter_time: # Save this slice to the grib file gribapi.grib_gribex_mode_off() if sample_file is not None and os.path.exists(sample_file): with open(sample_file, 'r') as f: grib_message = gribapi.grib_new_from_file(f) logger.info("Created grib message from file %s" % sample_file) else: logger.info("Creating grib message from gribapi sample: GRIB1") grib_message = gribapi.grib_new_from_samples("GRIB1") set_time(obj, grib_message) set_product(obj, grib_message) set_grid(obj, grib_message) set_data(obj, grib_message) gribapi.grib_write(grib_message, grib_file) gribapi.grib_release(grib_message) # if target was a string then we have to close the file we # created, otherwise leave that up to the user. if isinstance(target, basestring): grib_file.close()
def setUp(self): # Create a test object to stand in for a real PPField. self.grib_message = gribapi.grib_new_from_samples("GRIB2")