def read_metadata(data_dict, data_type): if data_type == 'VD': out = hdf_vd.get_metadata(data_dict[0]) elif data_type == 'SD': out = hdf_sd.get_metadata(data_dict[0]) else: raise ValueError("Invalid data-type: %s, HDF variables must be VD or SD only" % data_type) return out
def _create_cube(self, filenames, variable): import numpy as np from cis.data_io.hdf import _read_hdf4 from iris.cube import Cube, CubeList from iris.coords import DimCoord, AuxCoord from cis.time_util import calculate_mid_time, cis_standard_time_unit from cis.data_io.hdf_sd import get_metadata from cf_units import Unit variables = ['XDim', 'YDim', variable] logging.info("Listing coordinates: " + str(variables)) cube_list = CubeList() # Read each file individually, let Iris do the merging at the end. for f in filenames: sdata, vdata = _read_hdf4(f, variables) lat_coord = DimCoord(_get_MODIS_SDS_data(sdata['YDim']), standard_name='latitude', units='degrees') lon_coord = DimCoord(_get_MODIS_SDS_data(sdata['XDim']), standard_name='longitude', units='degrees') # create time coordinate using the midpoint of the time delta between the start date and the end date start_datetime = self._get_start_date(f) end_datetime = self._get_end_date(f) mid_datetime = calculate_mid_time(start_datetime, end_datetime) logging.debug("Using {} as datetime for file {}".format( mid_datetime, f)) time_coord = AuxCoord(mid_datetime, standard_name='time', units=cis_standard_time_unit, bounds=[start_datetime, end_datetime]) var = sdata[variable] metadata = get_metadata(var) try: units = Unit(metadata.units) except ValueError: logging.warning( "Unable to parse units '{}' in {} for {}.".format( metadata.units, f, variable)) units = None cube = Cube(_get_MODIS_SDS_data(sdata[variable]), dim_coords_and_dims=[(lon_coord, 1), (lat_coord, 0)], aux_coords_and_dims=[(time_coord, None)], var_name=metadata._name, long_name=metadata.long_name, units=units) cube_list.append(cube) # Merge the cube list across the scalar time coordinates before returning a single cube. return cube_list.merge_cube()
def read_metadata(data_dict, data_type): if data_type == 'VD': out = hdf_vd.get_metadata(data_dict[0]) elif data_type == 'SD': out = hdf_sd.get_metadata(data_dict[0]) else: raise ValueError( "Invalid data-type: %s, HDF variables must be VD or SD only" % data_type) return out
def hdf_read(filenames, variable, start=None, count=None, stride=None): """Returns variable, concatenated over a sequence of files.""" from cis.data_io.hdf import read from cis.data_io.hdf_sd import get_metadata from cis.utils import concatenate sdata, _ = read(filenames, variable) var = sdata[variable] data = concatenate( [_get_MODIS_SDS_data(i, start, count, stride) for i in var]) metadata = get_metadata(var[0]) return data, metadata
def ncdf_read(filenames, variable, start=None, count=None, stride=None): """Returns variable, concatenated over a sequence of files.""" from cis.data_io.netcdf import read, get_metadata from cis.utils import concatenate, listify data = [] for f in listify(filenames): sdata = read(f, variable) var = sdata[variable] data.append(_tidy_ncdf_data(var, start, count, stride)) metadata = get_metadata(var) return concatenate(data), metadata
def _create_cube(self, filenames, variable): import numpy as np from cis.data_io.hdf import _read_hdf4 from cis.data_io import hdf_vd from iris.cube import Cube, CubeList from iris.coords import DimCoord, AuxCoord from cis.time_util import calculate_mid_time, cis_standard_time_unit from cis.data_io.hdf_sd import get_metadata from cf_units import Unit variables = ['XDim:GlobalGrid', 'YDim:GlobalGrid', variable] logging.info("Listing coordinates: " + str(variables)) cube_list = CubeList() # Read each file individually, let Iris do the merging at the end. for f in filenames: sdata, vdata = _read_hdf4(f, variables) lat_points = np.linspace(-90., 90., hdf_vd.get_data(vdata['YDim:GlobalGrid'])) lon_points = np.linspace(-180., 180., hdf_vd.get_data(vdata['XDim:GlobalGrid'])) lat_coord = DimCoord(lat_points, standard_name='latitude', units='degrees') lon_coord = DimCoord(lon_points, standard_name='longitude', units='degrees') # create time coordinate using the midpoint of the time delta between the start date and the end date start_datetime = self._get_start_date(f) end_datetime = self._get_end_date(f) mid_datetime = calculate_mid_time(start_datetime, end_datetime) logging.debug("Using {} as datetime for file {}".format(mid_datetime, f)) time_coord = AuxCoord(mid_datetime, standard_name='time', units=cis_standard_time_unit, bounds=[start_datetime, end_datetime]) var = sdata[variable] metadata = get_metadata(var) try: units = Unit(metadata.units) except ValueError: logging.warning("Unable to parse units '{}' in {} for {}.".format(metadata.units, f, variable)) units = None cube = Cube(_get_MODIS_SDS_data(sdata[variable]), dim_coords_and_dims=[(lon_coord, 1), (lat_coord, 0)], aux_coords_and_dims=[(time_coord, None)], var_name=metadata._name, long_name=metadata.long_name, units=units) cube_list.append(cube) # Merge the cube list across the scalar time coordinates before returning a single cube. return cube_list.merge_cube()
def test_that_can_get_metadata_for_known_variable(): data_dict = hdf_sd.read(escape_colons(valid_hdf_sd_file)) metadata = hdf_sd.get_metadata(data_dict['Latitude']) eq_(metadata._name, "Latitude") eq_(metadata.standard_name, "latitude") eq_(metadata.long_name, "Geodetic Latitude") eq_(metadata.shape, [203, 135]) eq_(metadata.units, "Degrees_north") eq_(metadata.factor, 1.0) eq_(metadata.offset, 0.0) eq_(metadata.missing_value, -999.0) attr = metadata.misc eq_(len(attr), 5) eq_(attr['Parameter_Type'], "MODIS Input") eq_(attr['valid_range'], [-90.0, 90.0])
def test_that_can_get_metadata_for_known_variable(): data_dict = hdf_sd.read(valid_hdf_sd_file) metadata = hdf_sd.get_metadata(data_dict['Latitude']) eq_(metadata._name, "Latitude") eq_(metadata.standard_name, "latitude") eq_(metadata.long_name, "Geodetic Latitude") eq_(metadata.shape, [203, 135]) eq_(metadata.units, "Degrees_north") eq_(metadata.range, [-90.0, 90.0]) eq_(metadata.factor, 1.0) eq_(metadata.offset, 0.0) eq_(metadata.missing_value, -999.0) attr = metadata.misc eq_(len(attr), 10) eq_(attr['_FillValue'], -999.0) eq_(attr['Parameter_Type'], "MODIS Input")