def _create_coord_list(self): from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import Metadata from cis.time_util import cis_standard_time_unit as cstu # These implement a lot of what is necessary, but aren't in CIS style from acp_utils import rolling_window from orbit import ATSR lat_data = [] lon_data = [] time_data = [] for fname in self.filenames: prod = ATSR(fname) lat_data.append(prod.lat) lon_data.append(prod.lon) time_data.append(prod.get_time()) # TODO: Properly define metadata lat_meta = Metadata(standard_name="latitude", units="degrees") lon_meta = Metadata(standard_name="longitude", units="degrees") time_meta = Metadata(standard_name="time", units=cstu) lat = Coord(concatenate(lat_data), lat_meta, "Y") lat.update_shape() lat.update_range() lon = Coord(concatenate(lon_data), lon_meta, "Y") lon.update_shape() lon.update_range() time = Coord(concatenate(time_data), time_meta, "T") time.update_shape() time.update_range() return CoordList([lat, lon, time])
def _create_coord_list(self, filename): import numpy as np coords = CoordList() time_data = read(filename, 'time')['time'] try: alt_data = read(filename, 'altitude')['altitude'] except InvalidVariableError: alt_data = read(filename, 'range')['range'] len_y = alt_data.shape[1] time_arr = utils.expand_1d_to_2d_array(time_data[:], len_y, axis=1) t_coord = Coord(time_arr, get_metadata(time_data), axis='x') t_coord.convert_to_std_time() coords.append(t_coord) #alt_arr = utils.expand_1d_to_2d_array(alt_data[:], len_x, axis=0) alt_arr = alt_data[:, :, 0] #eliminate "angle" axis #alt_arr = alt_data #eliminate "angle" axis coords.append(Coord(alt_arr, get_metadata(alt_data), axis='y')) lat_data = read(filename, 'latitude')['latitude'] lat_arr = np.ones(alt_arr.shape) * lat_data[:] coords.append(Coord(lat_arr, get_metadata(lat_data))) lon_data = read(filename, 'longitude')['longitude'] lon_arr = np.ones(alt_arr.shape) * lon_data[:] coords.append(Coord(lon_arr, get_metadata(lon_data))) return coords
def _create_coord_list(self, filenames): from cis.data_io.netcdf import read_many_files_individually, get_metadata from cis.data_io.Coord import Coord from cis.exceptions import InvalidVariableError try: variables = ["lon", "lat", "time"] data = read_many_files_individually(filenames, variables) except InvalidVariableError: variables = ["longitude", "latitude", "time"] data = read_many_files_individually(filenames, variables) logging.info("Listing coordinates: " + str(variables)) coords = CoordList() coords.append( Coord(data[variables[0]], get_metadata(data[variables[0]][0]), "X")) coords.append( Coord(data[variables[1]], get_metadata(data[variables[1]][0]), "Y")) coords.append( self._fix_time( Coord(data[variables[2]], get_metadata(data[variables[2]][0]), "T"))) return coords
def setUp(self): x_points = np.arange(-10, 11, 5) y_points = np.arange(-5, 6, 5) y, x = np.meshgrid(y_points, x_points) x = Coord( x, Metadata(name='lat', standard_name='latitude', units='degrees')) y = Coord( y, Metadata(name='lon', standard_name='longitude', units='degrees')) data = np.reshape(np.arange(15) + 1.0, (5, 3)) self.coords = CoordList([x, y]) ug1 = UngriddedData( data, Metadata(standard_name='rainfall_flux', long_name="TOTAL RAINFALL RATE: LS+CONV KG/M2/S", units="kg m-2 s-1", missing_value=-999), self.coords) ug2 = UngriddedData( data * 0.1, Metadata(standard_name='snowfall_flux', long_name="TOTAL SNOWFALL RATE: LS+CONV KG/M2/S", units="kg m-2 s-1", missing_value=-999), self.coords) self.ungridded_data_list = UngriddedDataList([ug1, ug2])
def _create_coord_list(self): """Read file coordinates into a CIS object""" from cis.data_io.Coord import Coord, CoordList from reame.utils import ncdf_read try: lon_data, lon_metadata = ncdf_read(self.filenames, "longitude") lat_data, lat_metadata = ncdf_read(self.filenames, "latitude") except IndexError: lon_data, lon_metadata = ncdf_read(self.filenames, "lon") lat_data, lat_metadata = ncdf_read(self.filenames, "lat") lat = Coord(lat_data, lat_metadata, "Y") lat.update_shape() lat.update_range() lon = Coord(lon_data, lon_metadata, "X") lon.update_shape() lat.update_range() time_data, time_metadata = ncdf_read(self.filenames, "time") # Ensure the standard name is set time_metadata.standard_name = "time" time = Coord(time_data, time_metadata, "T") time.convert_TAI_time_to_std_time(ATSR_REFERENCE_TIME) time.update_shape() time.update_range() return CoordList([lat, lon, time])
def create_dummy_coordinates_list(): coord1 = Coord(numpy.array([5, 4]), Metadata(standard_name='grid_latitude'), axis='Y') coord2 = Coord(numpy.array([5, 4]), Metadata(standard_name='grid_longitude'), axis='X') return CoordList([coord1, coord2])
def _create_coord_list(self): from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import Metadata from cis.time_util import convert_sec_since_to_std_time from os.path import basename lat_all = [] lon_all = [] time_all = [] for fname in self.filenames: var_name = self.gdal_variable_name(fname, "Optical_Depth_055") if self.grid_path: granule = basename(fname).split(".")[2] lat_data, lon_data = self._read_grid_centres(granule) else: lat_data, lon_data = self._calculate_grid_centres(var_name) time_data = self._calculate_grid_time(var_name, lat_data, lon_data) # Workaround files containing only one day sh = (-1, ) + lat_data.shape time_data = time_data.reshape(sh) keep = np.logical_not(self._read_qcmask(fname)).reshape(sh) for time_slice, keep_slice in zip(time_data, keep): lat_all.extend(lat_data[keep_slice]) lon_all.extend(lon_data[keep_slice]) time_all.extend(time_slice[keep_slice]) if len(lat_all) == 0: raise NotImplementedError("It's empty!") lat = Coord( np.ma.array(lat_all), Metadata(name="lat", standard_name="latitude", units="degrees", range=(-90., 90.)), "Y") lat.update_shape() lon = Coord( np.ma.array(lon_all), Metadata(name="lon", standard_name="longitude", units="degrees", range=(-180., 180.)), "X") lon.update_shape() time = Coord( np.ma.array(time_all), Metadata(name="time", standard_name="time", units="Seconds since 1993-1-1 00:00:00.0 0"), "T") time.convert_TAI_time_to_std_time(MODIS_REFERENCE_TIME) time.update_shape() # Set the QC mask as we now know how many points we have self._qcmask = np.full(lat.shape, False) return CoordList([lat, lon, time])
def test_order_of_coords_doesnt_matter(self): from iris.cube import Cube from iris.coords import DimCoord import numpy as np from cis.data_io.gridded_data import make_from_cube from cis.data_io.ungridded_data import UngriddedCoordinates, Metadata from cis.data_io.Coord import Coord cube_lat = DimCoord(np.linspace(-90, 90, 18), standard_name='latitude', units='degrees') cube_lon = DimCoord(np.linspace(0, 359, 36), standard_name='longitude', units='degrees', circular=True) cube_alt = DimCoord(np.linspace(0, 10000, 10), standard_name='altitude', units='meters') times = np.linspace(0, 30, 12) cube_time = DimCoord(times, standard_name='time', units='days since 1970-01-01 00:00:00') data = np.arange(12 * 18 * 36 * 10).reshape(12, 18, 36, 10) source = make_from_cube( Cube(data, dim_coords_and_dims=[(cube_time, 0), (cube_lat, 1), (cube_lon, 2), (cube_alt, 3)])) n = 10 sample_lats = np.linspace(-9.1, 9.9, n) sample_longs = np.linspace(-9.1, 9.9, n) sample_alts = np.linspace(99, 599, n) sample_times = np.linspace(0, 30, n) sample = UngriddedCoordinates([ Coord(sample_lats, Metadata('latitude')), Coord(sample_longs, Metadata('longitude')), Coord(sample_alts, Metadata('altitude')), Coord(sample_times, Metadata('time', units='days since 1970-01-01 00:00:00')) ]) col = GriddedUngriddedCollocator() output = col.collocate(sample, source, None, 'nn')[0] source.transpose() col = GriddedUngriddedCollocator() assert_equal( col.collocate(sample, source, None, 'nn')[0].data, output.data) source.transpose((2, 1, 0, 3)) col = GriddedUngriddedCollocator() assert_equal( col.collocate(sample, source, None, 'nn')[0].data, output.data)
def setUp(self): x_points = np.arange(-10, 11, 5) y_points = np.arange(-5, 6, 5) y, x = np.meshgrid(y_points, x_points) self.x = Coord(x, Metadata(standard_name='latitude', units='degrees')) self.y = Coord(y, Metadata(standard_name='longitude', units='degrees')) coords = CoordList([self.x, self.y]) self.ug = UngriddedCoordinates(coords)
def can_find_many_coords_from_a_list_of_coordinates(): list = create_dummy_coordinates_list() list.append(Coord(numpy.array([5, 4]), Metadata(name='testZ'), axis='Z')) list.append( Coord(numpy.array([5, 4]), Metadata(name='testZ', standard_name='air_pressure'))) assert (len(list) == 4) coords = list.get_coords(var_name='testZ') assert (len(coords) == 2) assert (coords[0].axis == 'Z') assert (coords[1].axis == '') assert (coords[1].name() == 'air_pressure')
def _create_one_dimensional_coord_list(self, filenames, index_offset=1): """ Create a set of coordinates appropriate for a ond-dimensional (column integrated) variable :param filenames: :param int index_offset: For 5km products this will choose the coordinates which represent the start (0), middle (1) and end (2) of the 15 shots making up each column retrieval. :return: """ from pyhdf.error import HDF4Error from cis.data_io import hdf_sd import datetime as dt from cis.time_util import convert_sec_since_to_std_time, cis_standard_time_unit variables = ['Latitude', 'Longitude', "Profile_Time"] logging.info("Listing coordinates: " + str(variables)) # reading data from files sdata = {} for filename in filenames: try: sds_dict = hdf_sd.read(filename, variables) except HDF4Error as e: raise IOError(str(e)) for var in list(sds_dict.keys()): utils.add_element_to_list_in_dict(sdata, var, sds_dict[var]) # latitude lat_data = hdf.read_data(sdata['Latitude'], self._get_calipso_data)[:, index_offset] lat_metadata = hdf.read_metadata(sdata['Latitude'], "SD") lat_coord = Coord(lat_data, lat_metadata, 'Y') # longitude lon = sdata['Longitude'] lon_data = hdf.read_data(lon, self._get_calipso_data)[:, index_offset] lon_metadata = hdf.read_metadata(lon, "SD") lon_coord = Coord(lon_data, lon_metadata, 'X') # profile time, x time = sdata['Profile_Time'] time_data = hdf.read_data(time, self._get_calipso_data)[:, index_offset] time_data = convert_sec_since_to_std_time(time_data, dt.datetime(1993, 1, 1, 0, 0, 0)) time_coord = Coord(time_data, Metadata(name='Profile_Time', standard_name='time', shape=time_data.shape, units=cis_standard_time_unit), "T") # create the object containing all coordinates coords = CoordList() coords.append(lat_coord) coords.append(lon_coord) coords.append(time_coord) return coords
def scene_to_cis(scn): from cis.data_io.Coord import Coord from cis.data_io.ungridded_data import Metadata, UngriddedData data = scn['M05'] area = data.info['area'] units = data.info['units'] lats = Coord( area.lats, Metadata(standard_name='latitude', units='degrees'), 'y') lons = Coord( area.lons, Metadata(standard_name='longitude', units='degrees'), 'x') print(area.lons.info) ug = UngriddedData(data, Metadata(name=data.info['name'], units=units), [lats, lons]) return ug
def create_coords(self, filenames): from cis.data_io.netcdf import read_many_files_individually, get_metadata from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import UngriddedCoordinates var_data = read_many_files_individually(filenames, ["longitude", "latitude", "time"]) lon = Coord(var_data["longitude"], get_metadata(var_data["longitude"][0]), axis="x") lat = Coord(var_data["latitude"], get_metadata(var_data["latitude"][0]), axis="y") time = Coord(var_data["time"], get_metadata(var_data["time"][0]), axis="t") coords = CoordList([lat, lon, time]) return UngriddedCoordinates(coords)
def make_mock_CALIOP_data(data, name='', units=''): from cis.utils import expand_1d_to_2d_array vertical_levels, swath_length = data.shape lat = Coord( expand_1d_to_2d_array(np.arange(swath_length), vertical_levels), Metadata(standard_name='latitude')) lon = Coord( expand_1d_to_2d_array(np.arange(swath_length), vertical_levels), Metadata(standard_name='longitude')) alt = Coord( expand_1d_to_2d_array(np.arange(vertical_levels), swath_length, axis=1), Metadata(standard_name='altitude')) return UngriddedData(data, Metadata(name, units=units), [lat, lon, alt])
def _create_bounded_coord_list(self): from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import Metadata from cis.time_util import cis_standard_time_unit as cstu # These implement a lot of what is necessary, but aren't in CIS style from acp_utils import rolling_window from orbit import MODIS lat_data = [] lat_bounds = [] lon_data = [] lon_bounds = [] time_data = [] time_bounds = [] for fname in self.filenames: prod = MODIS(fname) lat_data.append(prod.lat) lon_data.append(prod.lon) lat_c = rolling_window(prod.lat_corner, (2, 2)) lat_bounds.append(lat_c.reshape(prod.shape + (4, ))) lon_c = rolling_window(prod.lon_corner, (2, 2)) lon_bounds.append(lon_c.reshape(prod.shape + (4, ))) t = prod.get_time() time_data.append(t) b = np.stack([t, np.roll(t, -1)], axis=2) b[-1, :, 1] = 2 * t[-1, :] - t[-2, :] time_bounds.append(b) # TODO: Properly define metadata lat_meta = Metadata(standard_name="latitude", units="degrees") lon_meta = Metadata(standard_name="longitude", units="degrees") time_meta = Metadata(standard_name="time", units=cstu) lat = Coord(concatenate(lat_data), lat_meta, "Y") lat.update_shape() lat.update_range() lat.bounds = concatenate(lat_bounds).reshape(lat.shape + (4, )) lat.bounds[..., 2:4] = lat.bounds[..., [3, 2]] lon = Coord(concatenate(lon_data), lon_meta, "Y") lon.update_shape() lon.update_range() lon.bounds = concatenate(lon_bounds).reshape(lon.shape + (4, )) lon.bounds[..., 2:4] = lon.bounds[..., [3, 2]] time = Coord(concatenate(time_data), time_meta, "T") time.update_shape() time.update_range() time.bounds = concatenate(time_bounds) return CoordList([lat, lon, time])
def _create_coord_list(self, filenames, data=None): if data is None: data = {} #initialise data dictionary inData = netCDF4.Dataset(filenames[0]) #open netCDF file data['longitude'] = np.array( inData.variables['longitude']) #extract longitudes data['latitude'] = np.array( inData.variables['latitude']) #extract latitudes origTimes = np.array(inData.variables['time']) #extract times #convert to days since 1600-01-01 (cis col doesn't work otherwise - not sure why...): niceDateTime = cf_units.num2date(origTimes, 'days since 1990-01-01 00:00:00', 'gregorian') data['time'] = cf_units.date2num(niceDateTime, 'days since 1600-01-01 00:00:00', 'gregorian') inData.close() #close netCDF file coords = CoordList() #initialise coordinate list #Append latitudes and longitudes to coordinate list: coords.append( Coord( data['longitude'], Metadata(name="longitude", long_name='longitude', standard_name='longitude', shape=(len(data), ), missing_value=-999.0, units="degrees_east", range=(-180, 180)), "x")) coords.append( Coord( data['latitude'], Metadata(name="latitude", long_name='latitude', standard_name='latitude', shape=(len(data), ), missing_value=-999.0, units="degrees_north", range=(-90, 90)), "y")) coords.append( Coord( data['time'], Metadata(name="time", long_name='time', standard_name='time', shape=(len(data), ), missing_value=-999.0, units="days since 1600-01-01 00:00:00"), "t")) return coords
def create_data_object(self, filenames, variable): from cis.data_io.netcdf import read_many_files_individually, get_metadata from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import UngriddedData var_data = read_many_files_individually(filenames, ["longitude", "latitude", "time", variable]) lon = Coord(var_data["longitude"], get_metadata(var_data["longitude"][0]), axis="x") lat = Coord(var_data["latitude"], get_metadata(var_data["latitude"][0]), axis="y") time = Coord(var_data["time"], get_metadata(var_data["time"][0]), axis="t") coords = CoordList([lat, lon, time]) usr_var_data = var_data[variable] return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords)
def create_data_object(self, filenames, variable): from pywork.CALIOP_utils import mask_data # Read all the vars without pressure var_data = super().create_data_object(filenames, variable) extinction_qc = super().create_data_object(filenames, "Extinction_QC_Flag_532") cad_score = super().create_data_object(filenames, "CAD_Score") qcd_var_data, = mask_data([var_data], cad_score, extinction_qc) if not variable.startswith('Column'): # Read pressure separately, as it's own variable pressure = super().create_data_object(filenames, "Pressure") qcd_pres, = mask_data([pressure], cad_score, extinction_qc) qcd_pres_coord = Coord(qcd_pres.data, qcd_pres.metadata, 'P') # Fix badly formatted units which aren't CF compliant and will break if they are aggregated if str(qcd_pres_coord.units) == "hPA": qcd_pres_coord.units = "hPa" qcd_var_data._coords.append(qcd_pres_coord) qcd_var_data._post_process() return qcd_var_data
def _add_aux_coordinate(dim_coords, filename, aux_coord_name, length): """ Add an auxiliary coordinate to a list of (reshaped) dimension coordinates :param dim_coords: CoordList of one-dimensional coordinates representing physical dimensions :param filename: The data file containing the aux coord :param aux_coord_name: The name of the aux coord to add to the coord list :param length: The length of the data dimensions which this auxiliary coordinate should span :return: A CoordList of reshaped (2D) physical coordinates plus the 2D auxiliary coordinate """ from cis.data_io.Coord import Coord from cis.utils import expand_1d_to_2d_array from cis.data_io.netcdf import read # We assume that the auxilliary coordinate is the same shape across files d = read(filename, [aux_coord_name])[aux_coord_name] # Reshape to the length given aux_data = expand_1d_to_2d_array(d[:], length, axis=0) # Get the length of the auxiliary coordinate len_y = d[:].size for dim_coord in dim_coords: dim_coord.data = expand_1d_to_2d_array(dim_coord.data, len_y, axis=1) all_coords = dim_coords + [Coord(aux_data, get_metadata(d))] return all_coords
def create_coords(self, filenames, variable=None): """ Reads the coordinates and data if required from the files :param filenames: List of filenames to read coordinates from :param variable: load a variable for the data :return: Coordinates """ from cis.data_io.netcdf import read_many_files_individually from cis.data_io.Coord import Coord, CoordList from cis.exceptions import InvalidVariableError variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"), ("time", "t"), ("air_pressure", "p")] dim_coords = CoordList() for v in variables: try: var_data = read_many_files_individually(filenames, v[0])[v[0]] dim_coords.append(Coord(var_data, get_metadata(var_data[0]), axis=v[1])) except InvalidVariableError: pass if variable is None: return UngriddedCoordinates(dim_coords) else: all_coords = self._add_aux_coordinate(dim_coords, filenames[0], 'DP_MID', dim_coords.get_coord(standard_name='time').data.size) usr_var_data = read_many_files_individually(filenames, variable)[variable] return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), all_coords)
def _create_time_coord(self, timestamp, time_variable_name, data_variables, coord_axis='T', standard_name='time'): """ Create a time coordinate, taking into account the fact that each file may have a different timestamp. :param timestamp: Timestamp or list of timestamps for :param time_variable_name: Name of the time variable :param data_variables: Dictionary containing one or multiple netCDF data variables for each variable name :param coord_axis: Axis, default 'T' :param standard_name: Coord standard name, default 'time' :return: Coordinate """ from cis.data_io.Coord import Coord from six.moves import zip_longest timestamps = listify(timestamp) time_variables = data_variables[time_variable_name] time_coords = [] # Create a coordinate for each separate file to account for differing timestamps for file_time_var, timestamp in zip_longest(time_variables, timestamps): metadata = get_metadata(file_time_var) metadata.standard_name = standard_name coord = Coord(file_time_var, metadata, coord_axis) coord.convert_to_std_time(timestamp) time_coords.append(coord) return Coord.from_many_coordinates(time_coords)
def create_coords(self, filenames, usr_variable=None): from cis.data_io.netcdf import read_many_files_individually, get_metadata from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData from cis.exceptions import InvalidVariableError variables = [("longitude", "x"), ("latitude", "y")] # if usr_variable is not None: # variables.append((usr_variable, '')) logging.info("Listing coordinates: " + str(variables)) coords = CoordList() var_data = read_many_files_individually(filenames, [v[0] for v in variables]) for var, (name, axis) in zip(var_data.values(), variables): try: coords.append(Coord(var, get_metadata(var[0]), axis=axis)) except InvalidVariableError: pass # Note - We don't need to convert this time coord as it should have been written in our # 'standard' time unit if usr_variable is None: res = UngriddedCoordinates(coords) else: usr_var_data = read_many_files_individually( filenames, usr_variable)[usr_variable] res = UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords) return res
def _create_fixed_value_coord(self, coord_axis, values, coord_units, points_counts, coord_name): """ Create a coordinate with a fixed value :param coord_axis: Axis of the coordinate in the coords :param coord_name: The name of the coordinate :param coord_units: The units for the coordinate :param points_counts: Number of points for this coordinate, or list of sizes for multiple files :param values: Value of coordinate, or list of values for multiple files :return: """ from cis.data_io.Coord import Coord values = listify(values) points_counts = listify(points_counts) all_points = np.array([]) # Create separate arrays with values and sizes corresponding to each of the different input files. for value, points_count in zip(values, points_counts): file_points = np.ma.array(np.zeros(points_count) + float(value)) all_points = np.append(all_points, file_points) metadata = Metadata(name=coord_name, shape=all_points.shape, units=coord_units, range=(min(values), max(values))) return Coord(all_points, metadata, coord_axis)
def can_not_convert_time_without_since_in_units_with_no_units(): times = numpy.array([0, 1]) units = "" time_stamp_info = "1601-01-01" coord = Coord(times, Metadata(units=units)) coord.convert_to_std_time(time_stamp_info)
def stack_data_list(data_list, var_name=None, units=None): """ Stacks a list of Ungridded data objects with the same data variable, but different coordinates into a single UngriddedData object, with accompanying lat, lon and time data. It assumes the coordinates have the same *sets* of coordinates, but that the coordinates themselves are different. :param data_list: list of UngriddedData objects to be merged (stacked vertically) :param string var_name: Name of final data variable :param string units: Units of final data variable :return: A merged UngriddedData object """ import numpy as np from cis.data_io.ungridded_data import UngriddedData, Metadata from cis.data_io.Coord import Coord coords = [] all_data = np.hstack((d.data_flattened for d in data_list)) # Create mask (not all datasets might be masked) mask = np.hstack((d.data.mask.ravel() if isinstance( d.data, np.ma.MaskedArray) else np.zeros(d.shape, np.bool) for d in data_list)) all_data = np.ma.MaskedArray(all_data, mask) for c in data_list[0].coords(): coord_data = np.hstack((d.coord(c).data_flattened for d in data_list)) coords.append(Coord(coord_data, c.metadata, axis=c.axis)) return UngriddedData( all_data, Metadata(name=var_name or data_list[0].name(), units=units or data_list[0].units), coords)
def _create_coord_list(self, filenames, variable=None): import datetime as dt from cis.time_util import convert_time_since_to_std_time, cis_standard_time_unit from cis.utils import concatenate from cf_units import Unit from geotiepoints import modis5kmto1km variables = ['Latitude', 'Longitude', 'View_time'] logging.info("Listing coordinates: " + str(variables)) sdata, vdata = hdf.read(filenames, variables) apply_interpolation = False if variable is not None: scale = self.__get_data_scale(filenames[0], variable) apply_interpolation = True if scale is "1km" else False lat_data = hdf.read_data(sdata['Latitude'], _get_MODIS_SDS_data) lat_metadata = hdf.read_metadata(sdata['Latitude'], "SD") lon_data = hdf.read_data(sdata['Longitude'], _get_MODIS_SDS_data) lon_metadata = hdf.read_metadata(sdata['Longitude'], "SD") if apply_interpolation: lon_data, lat_data = modis5kmto1km(lon_data, lat_data) lat_coord = Coord(lat_data, lat_metadata, 'Y') lon_coord = Coord(lon_data, lon_metadata, 'X') time = sdata['View_time'] time_metadata = hdf.read_metadata(time, "SD") # Ensure the standard name is set time_metadata.standard_name = 'time' time_metadata.units = cis_standard_time_unit t_arrays = [] for f, d in zip(filenames, time): time_start = self._get_start_date(f) t_data = _get_MODIS_SDS_data( d) / 24.0 # Convert hours since to days since t_offset = time_start - dt.datetime(1600, 1, 1) # Convert to CIS time t_arrays.append(t_data + t_offset.days) time_coord = Coord(concatenate(t_arrays), time_metadata, "T") return CoordList([lat_coord, lon_coord, time_coord])
def can_convert_time_with_since_in_units(): times = numpy.array([0, 1]) units = "days since 1601-01-01" coord = Coord(times, Metadata(units=units)) coord.convert_to_std_time() assert_that(coord.data_flattened[0], is_(366.0), "time point")
def test_GIVEN_missing_coord_values_WHEN_coords_THEN_missing_values_removed( self): x_points = np.arange(-10, 11, 5) y_points = np.arange(-5, 6, 5) y, x = np.meshgrid(y_points, x_points) y = np.ma.masked_array(y, np.zeros(y.shape, dtype=bool)) y.mask[1, 2] = True x = Coord(x, Metadata(standard_name='latitude', units='degrees')) y = Coord(y, Metadata(standard_name='longitude', units='degrees')) coords = CoordList([x, y]) ug = UngriddedCoordinates(coords) coords = ug.coords() for coord in coords: assert_that(len(coord.data), is_(14))
def test_GIVEN_missing_coord_values_WHEN_data_flattened_THEN_missing_values_removed( self): x_points = np.arange(-10, 11, 5) y_points = np.arange(-5, 6, 5) y, x = np.meshgrid(y_points, x_points) y = np.ma.masked_array(y, np.zeros(y.shape, dtype=bool)) y.mask[1, 2] = True x = Coord(x, Metadata(standard_name='latitude', units='degrees')) y = Coord(y, Metadata(standard_name='longitude', units='degrees')) coords = CoordList([x, y]) data = np.reshape(np.arange(15) + 1.0, (5, 3)) ug = UngriddedData(None, Metadata(), coords, lambda x: data) data = ug.data_flattened assert_that(len(data), is_(14))
def create_coords(self, filenames, usr_variable=None): from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData from cis.exceptions import InvalidVariableError # We have to read it once first to find out which variables are in there. We assume the set of coordinates in # all the files are the same file_variables = get_netcdf_file_variables(filenames[0]) def get_axis_std_name(var): axis=None lvar = var.lower() if lvar == 'longitude': axis = 'x', 'longitude' if lvar == 'latitude': axis = 'y', 'latitude' if lvar == 'G_ALT' or lvar == 'altitude' or lvar == 'pressure_altitude': axis = 'z', 'altitude' if lvar == 'time': axis = 't', 'time' if lvar == 'p' or lvar == 'pressure' or lvar == 'static_pressure': axis = 'p', 'air_pressure' return axis all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None] # Get rid of any duplicates coord_variables = [] for v in all_coord_variables: if v is None or v[1][1] not in [x[1][1] for x in coord_variables]: coord_variables.append(v) all_variables = coord_variables.copy() if usr_variable is not None: all_variables.append((usr_variable, '')) logging.info("Listing coordinates: " + str(all_variables)) coords = CoordList() var_data = read_many_files_individually(filenames, [v[0] for v in all_variables]) for name, axis_std_name in coord_variables: try: meta = get_metadata(var_data[name][0]) if meta.standard_name is None: meta.standard_name = axis_std_name[1] coords.append(Coord(var_data[name], meta, axis=axis_std_name[0])) except InvalidVariableError: pass # Note - We don't need to convert this time coord as it should have been written in our # 'standard' time unit if usr_variable is None: res = UngriddedCoordinates(coords) else: res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords) return res