def create_coords(self, filenames, usr_variable=None): from cis.data_io.netcdf import read_many_files_individually, get_metadata from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData from cis.exceptions import InvalidVariableError variables = [("lon", "x"), ("lat", "y")] # if usr_variable is not None: # variables.append((usr_variable, '')) logging.info("Listing coordinates: " + str(variables)) coords = CoordList() var_data = read_many_files_individually(filenames, [v[0] for v in variables]) for var, (name, axis) in zip(var_data.values(), variables): try: coords.append(Coord(var, get_metadata(var[0]), axis=axis)) except InvalidVariableError: pass # Note - We don't need to convert this time coord as it should have been written in our # 'standard' time unit if usr_variable is None: res = UngriddedCoordinates(coords) else: usr_var_data = read_many_files_individually(filenames, usr_variable)[usr_variable] res = UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords) return res
def create_coords(self, filenames, usr_variable=None): from cis.data_io.netcdf import read_many_files_individually, get_metadata from cis.data_io.Coord import Coord from cis.exceptions import InvalidVariableError variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"), ("time", "t"), ("air_pressure", "p")] logging.info("Listing coordinates: " + str(variables)) coords = CoordList() for variable in variables: try: var_data = read_many_files_individually(filenames, variable[0])[variable[0]] coords.append(Coord(var_data, get_metadata(var_data[0]), axis=variable[1])) except InvalidVariableError: pass # Note - We don't need to convert this time coord as it should have been written in our # 'standard' time unit if usr_variable is None: res = UngriddedCoordinates(coords) else: usr_var_data = read_many_files_individually(filenames, usr_variable)[usr_variable] res = UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords) return res
def create_coords(self, filenames, usr_variable=None): from cis.data_io.netcdf import read_many_files_individually, get_metadata from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData from cis.exceptions import InvalidVariableError variables = [("longitude", "x"), ("latitude", "y")] # if usr_variable is not None: # variables.append((usr_variable, '')) logging.info("Listing coordinates: " + str(variables)) coords = CoordList() var_data = read_many_files_individually(filenames, [v[0] for v in variables]) for var, (name, axis) in zip(var_data.values(), variables): try: coords.append(Coord(var, get_metadata(var[0]), axis=axis)) except InvalidVariableError: pass # Note - We don't need to convert this time coord as it should have been written in our # 'standard' time unit if usr_variable is None: res = UngriddedCoordinates(coords) else: usr_var_data = read_many_files_individually( filenames, usr_variable)[usr_variable] res = UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords) return res
def create_coords(self, filenames, variable=None): """ Reads the coordinates and data if required from the files :param filenames: List of filenames to read coordinates from :param variable: load a variable for the data :return: Coordinates """ from cis.data_io.netcdf import read_many_files_individually from cis.data_io.Coord import Coord, CoordList from cis.exceptions import InvalidVariableError variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"), ("time", "t"), ("air_pressure", "p")] dim_coords = CoordList() for v in variables: try: var_data = read_many_files_individually(filenames, v[0])[v[0]] dim_coords.append(Coord(var_data, get_metadata(var_data[0]), axis=v[1])) except InvalidVariableError: pass if variable is None: return UngriddedCoordinates(dim_coords) else: all_coords = self._add_aux_coordinate(dim_coords, filenames[0], 'DP_MID', dim_coords.get_coord(standard_name='time').data.size) usr_var_data = read_many_files_individually(filenames, variable)[variable] return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), all_coords)
def create_coords(self, filenames, usr_variable=None): from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData from cis.exceptions import InvalidVariableError # We have to read it once first to find out which variables are in there. We assume the set of coordinates in # all the files are the same file_variables = get_netcdf_file_variables(filenames[0]) def get_axis_std_name(var): axis=None lvar = var.lower() if lvar.startswith('lon'): axis = 'x', 'longitude' if lvar.startswith('lat'): axis = 'y', 'latitude' if lvar == 'G_ALT' or lvar == 'altitude' or lvar == 'pressure_altitude': axis = 'z', 'altitude' if lvar == 'time': axis = 't', 'time' if lvar == 'p' or lvar == 'pressure' or lvar == 'static_pressure': axis = 'p', 'air_pressure' return axis all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None] # Get rid of any duplicates coord_variables = [] for v in all_coord_variables: if v is None or v[1][1] not in [x[1][1] for x in coord_variables]: coord_variables.append(v) all_variables = coord_variables.copy() if usr_variable is not None: all_variables.append((usr_variable, '')) logging.info("Listing coordinates: " + str(all_variables)) coords = CoordList() var_data = read_many_files_individually(filenames, [v[0] for v in all_variables]) for name, axis_std_name in coord_variables: try: meta = get_metadata(var_data[name][0]) if meta.standard_name is None: meta.standard_name = axis_std_name[1] coord = Coord(var_data[name], meta, axis=axis_std_name[0]) if meta.standard_name == 'time': # Converting units to CIS std time coord.convert_to_std_time() coords.append(coord) except InvalidVariableError: pass if usr_variable is None: res = UngriddedCoordinates(coords) else: res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords) return res
def create_coords(self, filenames, usr_variable=None): from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData from cis.exceptions import InvalidVariableError # We have to read it once first to find out which variables are in there. We assume the set of coordinates in # all the files are the same file_variables = get_netcdf_file_variables(filenames[0]) def get_axis_std_name(var): axis=None lvar = var.lower() if lvar == 'longitude': axis = 'x', 'longitude' if lvar == 'latitude': axis = 'y', 'latitude' if lvar == 'G_ALT' or lvar == 'altitude' or lvar == 'pressure_altitude': axis = 'z', 'altitude' if lvar == 'time': axis = 't', 'time' if lvar == 'p' or lvar == 'pressure' or lvar == 'static_pressure': axis = 'p', 'air_pressure' return axis all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None] # Get rid of any duplicates coord_variables = [] for v in all_coord_variables: if v is None or v[1][1] not in [x[1][1] for x in coord_variables]: coord_variables.append(v) all_variables = coord_variables.copy() if usr_variable is not None: all_variables.append((usr_variable, '')) logging.info("Listing coordinates: " + str(all_variables)) coords = CoordList() var_data = read_many_files_individually(filenames, [v[0] for v in all_variables]) for name, axis_std_name in coord_variables: try: meta = get_metadata(var_data[name][0]) if meta.standard_name is None: meta.standard_name = axis_std_name[1] coords.append(Coord(var_data[name], meta, axis=axis_std_name[0])) except InvalidVariableError: pass # Note - We don't need to convert this time coord as it should have been written in our # 'standard' time unit if usr_variable is None: res = UngriddedCoordinates(coords) else: res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords) return res
def create_coords(self, filenames, usr_variable=None): from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData from cis.exceptions import InvalidVariableError # We have to read it once first to find out which variables are in there. We assume the set of coordinates in # all the files are the same file_variables = get_netcdf_file_variables(filenames[0]) def get_axis_std_name(lvar): axis=None if lvar == 'LON_JAVAD' or lvar == 'LON_OXTS': axis = 'x', 'longitude' if lvar == 'LAT_JAVAD' or lvar == 'LAT_OXTS': axis = 'y', 'latitude' if lvar == 'ALT_JAVAD' or lvar == 'ALT_OXTS': axis = 'z', 'altitude' if lvar == 'Time': axis = 't', 'time' if lvar == 'PS_AIR': axis = 'p', 'air_pressure' return axis all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None] # Get rid of any duplicates coord_variables = [] for v in all_coord_variables: if v is None or v[1][1] not in [x[1][1] for x in coord_variables]: coord_variables.append(v) all_variables = coord_variables.copy() if usr_variable is not None: all_variables.append((usr_variable, '')) logging.info("Listing coordinates: " + str(all_variables)) coords = CoordList() var_data = read_many_files_individually(filenames, [v[0] for v in all_variables]) for name, axis_std_name in coord_variables: try: meta = get_metadata(var_data[name][0]) if meta.standard_name is None: meta.standard_name = axis_std_name[1] coords.append(Coord(var_data[name], meta, axis=axis_std_name[0])) except InvalidVariableError: pass # Note - We don't need to convert this time coord as it should have been written in our # 'standard' time unit if usr_variable is None: res = UngriddedCoordinates(coords) else: res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords) return res
def _create_coord_list(self, filenames): import numpy as np from cis.time_util import calculate_mid_time, cis_standard_time_unit variables = ["XDim", "YDim"] logging.info("Listing coordinates: " + str(variables)) sdata, vdata = hdf.read(filenames, variables) lat = sdata["YDim"] lat_metadata = hdf.read_metadata(lat, "SD") lon = sdata["XDim"] lon_metadata = hdf.read_metadata(lon, "SD") # expand lat and lon data array so that they have the same shape lat_data = utils.expand_1d_to_2d_array( hdf.read_data(lat, "SD"), lon_metadata.shape, axis=1 ) # expand latitude column wise lon_data = utils.expand_1d_to_2d_array( hdf.read_data(lon, "SD"), lat_metadata.shape, axis=0 ) # expand longitude row wise lat_metadata.shape = lat_data.shape lon_metadata.shape = lon_data.shape # to make sure "Latitude" and "Longitude", i.e. the standard_name is displayed instead of "YDim"and "XDim" lat_metadata.standard_name = "latitude" lat_metadata._name = "" lon_metadata.standard_name = "longitude" lon_metadata._name = "" # create arrays for time coordinate using the midpoint of the time delta between the start date and the end date time_data_array = [] for filename in filenames: mid_datetime = calculate_mid_time(self._get_start_date(filename), self._get_end_date(filename)) logging.debug("Using " + str(mid_datetime) + " as datetime for file " + str(filename)) # Only use part of the full lat shape as it has already been concatenated time_data = np.empty((lat_metadata.shape[0] / len(filenames), lat_metadata.shape[1]), dtype="float64") time_data.fill(mid_datetime) time_data_array.append(time_data) time_data = utils.concatenate(time_data_array) time_metadata = Metadata( name="DateTime", standard_name="time", shape=time_data.shape, units=str(cis_standard_time_unit), calendar=cis_standard_time_unit.calendar, ) coords = CoordList() coords.append(Coord(lon_data, lon_metadata, "X")) coords.append(Coord(lat_data, lat_metadata, "Y")) coords.append(Coord(time_data, time_metadata, "T")) return coords
def _create_one_dimensional_coord_list(self, filenames, index_offset=1): """ Create a set of coordinates appropriate for a ond-dimensional (column integrated) variable :param filenames: :param int index_offset: For 5km products this will choose the coordinates which represent the start (0), middle (1) and end (2) of the 15 shots making up each column retrieval. :return: """ from pyhdf.error import HDF4Error from cis.data_io import hdf_sd import datetime as dt from cis.time_util import convert_sec_since_to_std_time, cis_standard_time_unit variables = ['Latitude', 'Longitude', "Profile_Time"] logging.info("Listing coordinates: " + str(variables)) # reading data from files sdata = {} for filename in filenames: try: sds_dict = hdf_sd.read(filename, variables) except HDF4Error as e: raise IOError(str(e)) for var in list(sds_dict.keys()): utils.add_element_to_list_in_dict(sdata, var, sds_dict[var]) # latitude lat_data = hdf.read_data(sdata['Latitude'], self._get_calipso_data)[:, index_offset] lat_metadata = hdf.read_metadata(sdata['Latitude'], "SD") lat_coord = Coord(lat_data, lat_metadata, 'Y') # longitude lon = sdata['Longitude'] lon_data = hdf.read_data(lon, self._get_calipso_data)[:, index_offset] lon_metadata = hdf.read_metadata(lon, "SD") lon_coord = Coord(lon_data, lon_metadata, 'X') # profile time, x time = sdata['Profile_Time'] time_data = hdf.read_data(time, self._get_calipso_data)[:, index_offset] time_data = convert_sec_since_to_std_time(time_data, dt.datetime(1993, 1, 1, 0, 0, 0)) time_coord = Coord(time_data, Metadata(name='Profile_Time', standard_name='time', shape=time_data.shape, units=cis_standard_time_unit), "T") # create the object containing all coordinates coords = CoordList() coords.append(lat_coord) coords.append(lon_coord) coords.append(time_coord) return coords
def create_coords(self, filenames, usr_variable=None): from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData from cis.exceptions import InvalidVariableError # We have to read it once first to find out which variables are in there. We assume the set of coordinates in # all the files are the same file_variables = get_netcdf_file_variables(filenames[0]) axis_lookup = { "longitude": "x", 'latitude': 'y', 'altitude': 'z', 'time': 't', 'air_pressure': 'p' } coord_variables = [(v, axis_lookup[v]) for v in file_variables if v in axis_lookup] # Create a copy to contain all the variables to read all_variables = list(coord_variables) if usr_variable is not None: all_variables.append((usr_variable, '')) logging.info("Listing coordinates: " + str(all_variables)) coords = CoordList() var_data = read_many_files_individually(filenames, [v[0] for v in all_variables]) for name, axis in coord_variables: try: coords.append( Coord(var_data[name], get_metadata(var_data[name][0]), axis=axis)) except InvalidVariableError: pass # Note - We don't need to convert this time coord as it should have been written in our # 'standard' time unit if usr_variable is None: res = UngriddedCoordinates(coords) else: res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords) return res
def _create_coord_list(self, filename): import numpy as np coords = CoordList() time_data = read(filename, 'time')['time'] try: alt_data = read(filename, 'altitude')['altitude'] except InvalidVariableError: alt_data = read(filename, 'range')['range'] len_y = alt_data.shape[1] time_arr = utils.expand_1d_to_2d_array(time_data[:], len_y, axis=1) t_coord = Coord(time_arr, get_metadata(time_data), axis='x') t_coord.convert_to_std_time() coords.append(t_coord) #alt_arr = utils.expand_1d_to_2d_array(alt_data[:], len_x, axis=0) alt_arr = alt_data[:, :, 0] #eliminate "angle" axis #alt_arr = alt_data #eliminate "angle" axis coords.append(Coord(alt_arr, get_metadata(alt_data), axis='y')) lat_data = read(filename, 'latitude')['latitude'] lat_arr = np.ones(alt_arr.shape) * lat_data[:] coords.append(Coord(lat_arr, get_metadata(lat_data))) lon_data = read(filename, 'longitude')['longitude'] lon_arr = np.ones(alt_arr.shape) * lon_data[:] coords.append(Coord(lon_arr, get_metadata(lon_data))) return coords
def _create_coord_list(self, filenames, data=None): from cis.data_io.ungridded_data import Metadata from cis.time_util import cis_standard_time_unit as ct if data is None: data_all = load_multiple_hysplit(filenames) # TODO only using first trajectory data = data_all[list(data_all.keys())[0]] coords = CoordList() latM = Metadata(name="Latitude", shape=(len(data),), units="degrees_north", range=(-90,90), standard_name='latitude') lonM = Metadata(name="Longitude", shape=(len(data),), units="degrees_east", range=(-180,180), standard_name='longitude') altM = Metadata(name="Altitude", shape=(len(data),), units="meters", standard_name='altitude') presM = Metadata(name="Pressure", shape=(len(data),), units="hPa", standard_name='air_pressure') timeM = Metadata(name="DateTime", standard_name="time", shape=(len(data),), units=str(ct)) coords.append(Coord(data['LAT'], latM)) coords.append(Coord(data['LON'], lonM)) coords.append(Coord(data['ALT'], altM)) coords.append(Coord(data['PRESSURE'], presM)) coords.append(Coord(data['DATETIMES'], timeM, "X")) # TODO Why X axis? return coords
def _create_coord_list(self, filename): import numpy as np coords = CoordList() time_data = read(filename, 'time')['time'] len_x = time_data.shape[0] try: alt_data = read(filename, 'altitude')['altitude'] except InvalidVariableError: alt_data = read(filename, 'range')['range'] len_y = alt_data.shape[0] time_arr = utils.expand_1d_to_2d_array(time_data[:], len_y, axis=1) t_coord = Coord(time_arr, get_metadata(time_data), axis='x') t_coord.convert_to_std_time() coords.append(t_coord) alt_arr = utils.expand_1d_to_2d_array(alt_data[:], len_x, axis=0) coords.append(Coord(alt_arr, get_metadata(alt_data), axis='y')) lat_data = read(filename, 'latitude')['latitude'] lat_arr = np.ones(alt_arr.shape) * lat_data[:] coords.append(Coord(lat_arr, get_metadata(lat_data))) lon_data = read(filename, 'longitude')['longitude'] lon_arr = np.ones(alt_arr.shape) * lon_data[:] coords.append(Coord(lon_arr, get_metadata(lon_data))) return coords
def _create_coordinates_list(self, data_variables, variable_selector): """ Create a co-ordinate list for the data :param data_variables: the load data :param variable_selector: the variable selector for the data :return: a list of coordinates """ coords = CoordList() # Time time_coord = self._create_time_coord( variable_selector.time_stamp_info, variable_selector.time_variable_name, data_variables) coords.append(time_coord) # Lat and Lon # Multiple points counts for multiple files points_count = [ np.product(var.shape) for var in data_variables[variable_selector.time_variable_name] ] if variable_selector.station: lat_coord = self._create_fixed_value_coord( "Y", variable_selector.station_latitude, "degrees_north", points_count, "latitude") lon_coord = self._create_fixed_value_coord( "X", variable_selector.station_longitude, "degrees_east", points_count, "longitude") else: lat_coord = self._create_coord( "Y", variable_selector.latitude_variable_name, data_variables, "latitude") lon_coord = self._create_coord( "X", variable_selector.longitude_variable_name, data_variables, "longitude") coords.append(lat_coord) coords.append(lon_coord) # Altitude if variable_selector.altitude is None: altitude_coord = self._create_coord( "Z", variable_selector.altitude_variable_name, data_variables, "altitude") else: altitude_coord = self._create_fixed_value_coord( "Z", variable_selector.altitude, "meters", points_count, "altitude") coords.append(altitude_coord) # Pressure if variable_selector.pressure_variable_name is not None: coords.append( self._create_coord("P", variable_selector.pressure_variable_name, data_variables, "air_pressure")) return coords
def _create_one_dimensional_coord_list(self, filenames): from cis.time_util import cis_standard_time_unit # list of coordinate variables we are interested in variables = [ 'MODIS_latitude', 'MODIS_longitude', 'TAI_start', 'Profile_time' ] # reading the various files logging.info("Listing coordinates: " + str(variables)) sdata, vdata = hdf.read(filenames, variables) # latitude lat = sdata['MODIS_latitude'] lat_data = hdf.read_data(lat, self._get_cloudsat_sds_data) lat_metadata = hdf.read_metadata(lat, "SD") lat_metadata.shape = lat_data.shape lat_metadata.standard_name = 'latitude' lat_coord = Coord(lat_data, lat_metadata) # longitude lon = sdata['MODIS_longitude'] lon_data = hdf.read_data(lon, self._get_cloudsat_sds_data) lon_metadata = hdf.read_metadata(lon, "SD") lon_metadata.shape = lon_data.shape lon_metadata.standard_name = 'longitude' lon_coord = Coord(lon_data, lon_metadata) # time coordinate time_data = self._generate_time_array(vdata) time_coord = Coord( time_data, Metadata(name='Profile_time', standard_name='time', shape=time_data.shape, units=cis_standard_time_unit), "X") # create object containing list of coordinates coords = CoordList() coords.append(lat_coord) coords.append(lon_coord) coords.append(time_coord) return coords
def create_coords(self, filenames, usr_variable=None): from cis.data_io.netcdf import read_many_files_individually, get_metadata from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData from cis.data_io.Coord import Coord, CoordList from cis.exceptions import InvalidVariableError variables = [("lon", "x", 'longitude'), ("lat", "y", 'latitude'), ("alt", "z", 'altitude'), ("time", "t", 'time'), ("p", "p", 'air_pressure')] logging.info("Listing coordinates: " + str(variables)) coords = CoordList() for variable in variables: try: var_data = read_many_files_individually( filenames, variable[0])[variable[0]] meta = get_metadata(var_data[0]) meta.standard_name = variable[2] # Some of the variables have an illegal name attribute... meta.misc.pop('name', None) c = Coord(var_data, meta, axis=variable[1]) if variable[1] == 'z': c.convert_units('m') coords.append(c) except InvalidVariableError: pass # Note - We don't need to convert this time coord as it should have been written in our # 'standard' time unit if usr_variable is None: res = UngriddedCoordinates(coords) else: usr_var_data = read_many_files_individually( filenames, usr_variable)[usr_variable] meta = get_metadata(usr_var_data[0]) # Some of the variables have an illegal name attribute... meta.misc.pop('name', None) res = UngriddedData(usr_var_data, meta, coords) return res
def _create_coord_list(self, filenames): from cis.data_io.netcdf import read_many_files_individually, get_metadata from cis.data_io.Coord import Coord variables = ["lat", "lon", "time"] logging.info("Listing coordinates: " + str(variables)) var_data = read_many_files_individually(filenames, variables) coords = CoordList() coords.append(Coord(var_data['lat'], get_metadata(var_data['lat'][0]), 'Y')) coords.append(Coord(var_data['lon'], get_metadata(var_data['lon'][0]), 'X')) time_coord = Coord(var_data['time'], get_metadata(var_data['time'][0])) # TODO: Is this really julian? time_coord.convert_julian_to_std_time() coords.append(time_coord) return coords
def _create_coord_list(self, filenames): from cis.data_io.netcdf import read_many_files, get_metadata from cis.data_io.Coord import Coord import datetime # FIXME: when reading an existing file variables might be "latitude", "longitude" variables = ["lat", "lon", "time"] logging.info("Listing coordinates: " + str(variables)) data = read_many_files(filenames, variables, dim="pixel_number") coords = CoordList() coords.append(Coord(data["lon"], get_metadata(data["lon"]), "X")) coords.append(Coord(data["lat"], get_metadata(data["lat"]), "Y")) time_coord = Coord(data["time"], get_metadata(data["time"]), "T") time_coord.convert_TAI_time_to_std_time(datetime.datetime(1970, 1, 1)) coords.append(time_coord) return coords
def _create_coord_list(self, filenames): from cis.data_io.netcdf import read_many_files_individually, get_metadata from cis.data_io.Coord import Coord from cis.exceptions import InvalidVariableError try: variables = ["lon", "lat", "time"] data = read_many_files_individually(filenames, variables) except InvalidVariableError: variables = ["longitude", "latitude", "time"] data = read_many_files_individually(filenames, variables) logging.info("Listing coordinates: " + str(variables)) coords = CoordList() coords.append(Coord(data[variables[0]], get_metadata(data[variables[0]][0]), "X")) coords.append(Coord(data[variables[1]], get_metadata(data[variables[1]][0]), "Y")) coords.append(self._fix_time(Coord(data[variables[2]], get_metadata(data[variables[2]][0]), "T"))) return coords
def _create_coord_list(self, filenames): from cis.time_util import cis_standard_time_unit # list of coordinate variables we are interested in variables = ['Latitude', 'Longitude', 'TAI_start', 'Profile_time', 'Height'] # reading the various files try: logging.info("Listing coordinates: " + str(variables)) sdata, vdata = hdf.read(filenames, variables) # altitude coordinate height = sdata['Height'] height_data = hdf.read_data(height, "SD") height_metadata = hdf.read_metadata(height, "SD") height_coord = Coord(height_data, height_metadata, "Y") except InvalidVariableError: # This means we are reading a Cloudsat file without height, so remove height from the variables list variables.remove('Height') logging.info("Listing coordinates: " + str(variables)) sdata, vdata = hdf.read(filenames, variables) height_data = None height_coord = None # latitude lat = vdata['Latitude'] lat_data = hdf.read_data(lat, "VD") if height_data is not None: lat_data = utils.expand_1d_to_2d_array(lat_data, len(height_data[0]), axis=1) lat_metadata = hdf.read_metadata(lat, "VD") lat_metadata.shape = lat_data.shape lat_coord = Coord(lat_data, lat_metadata) # longitude lon = vdata['Longitude'] lon_data = hdf.read_data(lon, "VD") if height_data is not None: lon_data = utils.expand_1d_to_2d_array(lon_data, len(height_data[0]), axis=1) lon_metadata = hdf.read_metadata(lon, "VD") lon_metadata.shape = lon_data.shape lon_coord = Coord(lon_data, lon_metadata) # time coordinate time_data = self._generate_time_array(vdata) if height_data is not None: time_data = utils.expand_1d_to_2d_array(time_data, len(height_data[0]), axis=1) time_coord = Coord(time_data, Metadata(name='Profile_time', standard_name='time', shape=time_data.shape, units=str(cis_standard_time_unit), calendar=cis_standard_time_unit.calendar), "X") # create object containing list of coordinates coords = CoordList() coords.append(lat_coord) coords.append(lon_coord) if height_coord is not None: coords.append(height_coord) coords.append(time_coord) return coords
def _create_coord_list(self, filenames): from cis.data_io.netcdf import read_many_files_individually, get_metadata from cis.data_io.Coord import Coord from cis.exceptions import InvalidVariableError try: variables = ["lon", "lat", "time"] data = read_many_files_individually(filenames, variables) except InvalidVariableError: variables = ["longitude", "latitude", "time"] data = read_many_files_individually(filenames, variables) logging.info("Listing coordinates: " + str(variables)) coords = CoordList() coords.append( Coord(data[variables[0]], get_metadata(data[variables[0]][0]), "X")) coords.append( Coord(data[variables[1]], get_metadata(data[variables[1]][0]), "Y")) coords.append( self._fix_time( Coord(data[variables[2]], get_metadata(data[variables[2]][0]), "T"))) return coords
def create_coords(self, filenames, usr_variable=None): from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData from cis.exceptions import InvalidVariableError # We have to read it once first to find out which variables are in there. We assume the set of coordinates in # all the files are the same file_variables = get_netcdf_file_variables(filenames[0]) axis_lookup = {"longitude": "x", 'latitude': 'y', 'altitude': 'z', 'time': 't', 'air_pressure': 'p'} coord_variables = [(v, axis_lookup[v]) for v in file_variables if v in axis_lookup] # Create a copy to contain all the variables to read all_variables = list(coord_variables) if usr_variable is not None: all_variables.append((usr_variable, '')) logging.info("Listing coordinates: " + str(all_variables)) coords = CoordList() var_data = read_many_files_individually(filenames, [v[0] for v in all_variables]) for name, axis in coord_variables: try: coords.append(Coord(var_data[name], get_metadata(var_data[name][0]), axis=axis)) except InvalidVariableError: pass # Note - We don't need to convert this time coord as it should have been written in our # 'standard' time unit if usr_variable is None: res = UngriddedCoordinates(coords) else: res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords) return res
def _create_one_dimensional_coord_list(self, filenames): from cis.time_util import cis_standard_time_unit # list of coordinate variables we are interested in variables = ['MODIS_latitude', 'MODIS_longitude', 'TAI_start', 'Profile_time'] # reading the various files logging.info("Listing coordinates: " + str(variables)) sdata, vdata = hdf.read(filenames, variables) # latitude lat = sdata['MODIS_latitude'] lat_data = hdf.read_data(lat, self._get_cloudsat_sds_data) lat_metadata = hdf.read_metadata(lat, "SD") lat_metadata.shape = lat_data.shape lat_metadata.standard_name = 'latitude' lat_coord = Coord(lat_data, lat_metadata) # longitude lon = sdata['MODIS_longitude'] lon_data = hdf.read_data(lon, self._get_cloudsat_sds_data) lon_metadata = hdf.read_metadata(lon, "SD") lon_metadata.shape = lon_data.shape lon_metadata.standard_name = 'longitude' lon_coord = Coord(lon_data, lon_metadata) # time coordinate time_data = self._generate_time_array(vdata) time_coord = Coord(time_data, Metadata(name='Profile_time', standard_name='time', shape=time_data.shape, units=cis_standard_time_unit), "X") # create object containing list of coordinates coords = CoordList() coords.append(lat_coord) coords.append(lon_coord) coords.append(time_coord) return coords
def create_coords(self, filenames, usr_variable=None): from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import UngriddedCoordinates from cis.exceptions import InvalidVariableError variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"), ("time", "t"), ("relative_humidity", "RH"), ("surface_air_pressure", "Pa"), ("air_temprature", "K"), ("wind_speed"), ("Wind Diretion"), ("rainfall_rate")] logging.info("Listing coordinates: " + str(variables)) coords = CoordList() for variable in variables: try: var_data = read_many_files_individually( filenames, variable[0])[variable[0]] coords.append( Coord(var_data, get_metadata(var_data[0]), axis=variable[1])) except InvalidVariableError: pass return UngriddedCoordinates(coords)
def create_coords(self, filenames, usr_variable=None): from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import UngriddedCoordinates from cis.exceptions import InvalidVariableError variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"), ("time", "t"), ("aerosol_backscatter_coefficient", "m-1 sr-1")] logging.info("Listing coordinates: " + str(variables)) coords = CoordList() for variable in variables: try: var_data = read_many_files_individually( filenames, variable[0])[variable[0]] coords.append( Coord(var_data, get_metadata(var_data[0]), axis=variable[1])) except InvalidVariableError: pass return UngriddedCoordinates(coords)
def _create_coord_list(self, filenames, data=None): from cis.data_io.ungridded_data import Metadata from cis.data_io.aeronet import load_multiple_aeronet from cis.time_util import cis_standard_time_unit as ct if data is None: data = load_multiple_aeronet(filenames) coords = CoordList() coords.append(Coord(data['longitude'], Metadata(name="Longitude", shape=(len(data),), units="degrees_east", range=(-180, 180)))) coords.append(Coord(data['latitude'], Metadata(name="Latitude", shape=(len(data),), units="degrees_north", range=(-90, 90)))) coords.append(Coord(data['altitude'], Metadata(name="Altitude", shape=(len(data),), units="meters"))) coords.append(Coord(data["datetime"], Metadata(name="DateTime", standard_name='time', shape=(len(data),), units=str(ct)), "X")) return coords
def _create_coord_list(self, filenames, data=None): from cis.data_io.ungridded_data import Metadata from cis.data_io.aeronet import load_multiple_aeronet from cis.time_util import cis_standard_time_unit as ct if data is None: data = load_multiple_aeronet(filenames) coords = CoordList() coords.append(Coord(data['longitude'], Metadata(name="Longitude", shape=(len(data),), units="degrees_east", range=(-180, 180)))) coords.append(Coord(data['latitude'], Metadata(name="Latitude", shape=(len(data),), units="degrees_north", range=(-90, 90)))) coords.append(Coord(data['altitude'], Metadata(name="Altitude", shape=(len(data),), units="meters"))) coords.append(Coord(data["datetime"], Metadata(name="DateTime", standard_name='time', shape=(len(data),), units=ct), "X")) return coords
def create_coords(self, filenames, variable=None): from cis.data_io.ungridded_data import Metadata from numpy import genfromtxt, NaN from cis.exceptions import InvalidVariableError from cis.time_util import convert_datetime_to_std_time import dateutil.parser as du array_list = [] for filename in filenames: try: array_list.append(genfromtxt(filename, dtype="f8,f8,f8,O,f8", names=['latitude', 'longitude', 'altitude', 'time', 'value'], delimiter=',', missing_values='', usemask=True, invalid_raise=True, converters={"time": du.parse})) except: raise IOError('Unable to read file ' + filename) data_array = utils.concatenate(array_list) n_elements = len(data_array['latitude']) coords = CoordList() coords.append(Coord(data_array["latitude"], Metadata(standard_name="latitude", shape=(n_elements,), units="degrees_north"))) coords.append(Coord(data_array["longitude"], Metadata(standard_name="longitude", shape=(n_elements,), units="degrees_east"))) coords.append( Coord(data_array["altitude"], Metadata(standard_name="altitude", shape=(n_elements,), units="meters"))) time_arr = convert_datetime_to_std_time(data_array["time"]) time = Coord(time_arr, Metadata(standard_name="time", shape=(n_elements,), units="days since 1600-01-01 00:00:00")) coords.append(time) if variable: try: data = UngriddedData(data_array['value'], Metadata(name="value", shape=(n_elements,), units="unknown", missing_value=NaN), coords) except: InvalidVariableError("Value column does not exist in file " + filenames) return data else: return UngriddedCoordinates(coords)
def _create_coordinates_list(self, data_variables, variable_selector): """ Create a co-ordinate list for the data :param data_variables: the load data :param variable_selector: the variable selector for the data :return: a list of coordinates """ coords = CoordList() # Time time_coord = self._create_time_coord(variable_selector.time_stamp_info, variable_selector.time_variable_name, data_variables) coords.append(time_coord) # Lat and Lon # Multiple points counts for multiple files points_count = [np.product(var.shape) for var in data_variables[variable_selector.time_variable_name]] if variable_selector.station: lat_coord = self._create_fixed_value_coord("Y", variable_selector.station_latitude, "degrees_north", points_count, "latitude") lon_coord = self._create_fixed_value_coord("X", variable_selector.station_longitude, "degrees_east", points_count, "longitude") else: lat_coord = self._create_coord("Y", variable_selector.latitude_variable_name, data_variables, "latitude") lon_coord = self._create_coord("X", variable_selector.longitude_variable_name, data_variables, "longitude") coords.append(lat_coord) coords.append(lon_coord) # Altitude if variable_selector.altitude is None: altitude_coord = self._create_coord("Z", variable_selector.altitude_variable_name, data_variables, "altitude") else: altitude_coord = self._create_fixed_value_coord("Z", variable_selector.altitude, "meters", points_count, "altitude") coords.append(altitude_coord) # Pressure if variable_selector.pressure_variable_name is not None: coords.append( self._create_coord("P", variable_selector.pressure_variable_name, data_variables, "air_pressure")) return coords
def _create_coord_list(self, filenames, data=None): if data is None: data = {} #initialise data dictionary inData = netCDF4.Dataset(filenames[0]) #open netCDF file data['longitude'] = np.array( inData.variables['longitude']) #extract longitudes data['latitude'] = np.array( inData.variables['latitude']) #extract latitudes origTimes = np.array(inData.variables['time']) #extract times #convert to days since 1600-01-01 (cis col doesn't work otherwise - not sure why...): niceDateTime = cf_units.num2date(origTimes, 'days since 1990-01-01 00:00:00', 'gregorian') data['time'] = cf_units.date2num(niceDateTime, 'days since 1600-01-01 00:00:00', 'gregorian') inData.close() #close netCDF file coords = CoordList() #initialise coordinate list #Append latitudes and longitudes to coordinate list: coords.append( Coord( data['longitude'], Metadata(name="longitude", long_name='longitude', standard_name='longitude', shape=(len(data), ), missing_value=-999.0, units="degrees_east", range=(-180, 180)), "x")) coords.append( Coord( data['latitude'], Metadata(name="latitude", long_name='latitude', standard_name='latitude', shape=(len(data), ), missing_value=-999.0, units="degrees_north", range=(-90, 90)), "y")) coords.append( Coord( data['time'], Metadata(name="time", long_name='time', standard_name='time', shape=(len(data), ), missing_value=-999.0, units="days since 1600-01-01 00:00:00"), "t")) return coords
def _create_coord_list(self, filenames, data=None): if data is None: data = {} #initialise data dictionary inData = netCDF4.Dataset(filenames[0]) #open netCDF file data['longitude'] = np.array(inData.variables['lon']) #extract longitudes data['latitude'] = np.array(inData.variables['lat']) #extract latitudes origTimes = np.array(inData.variables['time_counter']) #extract times #Convert time to days since niceDateTime = cf_units.num2date(origTimes,'seconds since 1999-01-01 00:00:00', 'gregorian') data['time_counter'] = cf_units.date2num(niceDateTime,'days since 1600-01-01 00:00:00', 'gregorian') inData.close() #close netCDF file coords = CoordList() #initialise coordinate list #Append latitudes and longitudes to coordinate list: coords.append(Coord(data['longitude'],Metadata(name="longitude",long_name='longitude',standard_name='longitude',shape=(len(data),),missing_value=-999.0,units="degrees_east",range=(-180, 180)),"x")) coords.append(Coord(data['latitude'],Metadata(name="latitude",long_name='latitude',standard_name='latitude',shape=(len(data),),missing_value=-999.0,units="degrees_north",range=(-90, 90)),"y")) coords.append(Coord(data['time'],Metadata(name="time",long_name='time',standard_name='time',shape=(len(data),),missing_value=-999.0,units="days since 1600-01-01 00:00:00"),"t")) return coords
def _create_coord_list(self, filenames, data=None): from cis.data_io.ungridded_data import Metadata from cis.time_util import cis_standard_time_unit as ct import numpy as np if data is None: data = load_multiple_hysplit(filenames) # TODO error handling coords = CoordList() #print(data['DATETIMES']) latM = Metadata(standard_name="latitude", shape=(len(data['LAT']), ), units="degrees_north", range=(-90, 90)) lonM = Metadata(standard_name="longitude", shape=(len(data['LON']), ), units="degrees_east", range=(-180, 180)) altM = Metadata(standard_name="altitude", shape=(len(data['ALT']), ), units="m") timeM = Metadata(standard_name="time", shape=(len(data['DATETIMES']), ), units=str(ct)) #timeM = Metadata(name="DateTime", standard_name="time", shape=(len(data['DATETIMES']),), units=str(ct)) pressM = Metadata(standard_name="air_pressure", shape=(len(data['PRESSURE']), ), units="Pa") #start_timeM = Metadata(name="start_time", standard_name="forecast_reference_time", shape=(len(data['STARTING_TIME']),), units=str(ct)) #start_heightM = Metadata(name="start_height", shape=(len(data['STARTING_HEIGHT']),), units="meters") #station_noM = Metadata(name="station_no", standard_name="institution", shape=(len(data['STATION_NO']),)) coords.append(Coord(data['DATETIMES'], timeM)) coords.append(Coord(data['PRESSURE'], pressM)) coords.append(Coord(data['LAT'], latM)) coords.append(Coord(data['LON'], lonM)) coords.append(Coord(data['ALT'], altM)) #coords.append(Coord(data['STARTING_TIME'], start_timeM)) #coords.append(Coord(data['STARTING_HEIGHT'], start_heightM)) #coords.append(Coord(data['STATION_NO'], station_noM)) return coords
def _create_coord_list(self, filenames): from cis.time_util import cis_standard_time_unit # list of coordinate variables we are interested in variables = [ 'Latitude', 'Longitude', 'TAI_start', 'Profile_time', 'Height' ] # reading the various files try: logging.info("Listing coordinates: " + str(variables)) sdata, vdata = hdf.read(filenames, variables) # altitude coordinate height = sdata['Height'] height_data = hdf.read_data(height, self._get_cloudsat_sds_data) height_metadata = hdf.read_metadata(height, "SD") height_coord = Coord(height_data, height_metadata, "Y") except InvalidVariableError: # This means we are reading a Cloudsat file without height, so remove height from the variables list variables.remove('Height') logging.info("Listing coordinates: " + str(variables)) sdata, vdata = hdf.read(filenames, variables) height_data = None height_coord = None # latitude lat = vdata['Latitude'] lat_data = hdf.read_data(lat, self._get_cloudsat_vds_data) if height_data is not None: lat_data = utils.expand_1d_to_2d_array(lat_data, len(height_data[0]), axis=1) lat_metadata = hdf.read_metadata(lat, "VD") lat_metadata.shape = lat_data.shape lat_coord = Coord(lat_data, lat_metadata) # longitude lon = vdata['Longitude'] lon_data = hdf.read_data(lon, self._get_cloudsat_vds_data) if height_data is not None: lon_data = utils.expand_1d_to_2d_array(lon_data, len(height_data[0]), axis=1) lon_metadata = hdf.read_metadata(lon, "VD") lon_metadata.shape = lon_data.shape lon_coord = Coord(lon_data, lon_metadata) # time coordinate time_data = self._generate_time_array(vdata) if height_data is not None: time_data = utils.expand_1d_to_2d_array(time_data, len(height_data[0]), axis=1) time_coord = Coord( time_data, Metadata(name='Profile_time', standard_name='time', shape=time_data.shape, units=cis_standard_time_unit), "X") # create object containing list of coordinates coords = CoordList() coords.append(lat_coord) coords.append(lon_coord) if height_coord is not None: coords.append(height_coord) coords.append(time_coord) return coords
def _create_coord_list(self, filenames, index_offset=0): import logging from cis.data_io import hdf as hdf from cis.data_io.Coord import Coord, CoordList from cis.data_io.ungridded_data import Metadata import cis.utils as utils from cis.data_io.hdf_vd import VDS from pyhdf.error import HDF4Error from cis.data_io import hdf_sd import datetime as dt from cis.time_util import convert_sec_since_to_std_time, cis_standard_time_unit variables = ['Latitude', 'Longitude', "Profile_Time", "Pressure"] logging.info("Listing coordinates: " + str(variables)) # reading data from files sdata = {} for filename in filenames: try: sds_dict = hdf_sd.read(filename, variables) except HDF4Error as e: raise IOError(str(e)) for var in list(sds_dict.keys()): utils.add_element_to_list_in_dict(sdata, var, sds_dict[var]) alt_name = "altitude" logging.info("Additional coordinates: '" + alt_name + "'") # work out size of data arrays # the coordinate variables will be reshaped to match that. # NOTE: This assumes that all Caliop_L1 files have the same altitudes. # If this is not the case, then the following line will need to be changed # to concatenate the data from all the files and not just arbitrarily pick # the altitudes from the first file. alt_data = get_data(VDS(filenames[0], "Lidar_Data_Altitudes"), True) alt_data *= 1000.0 # Convert to m len_x = alt_data.shape[0] lat_data = hdf.read_data(sdata['Latitude'], self._get_calipso_data) len_y = lat_data.shape[0] new_shape = (len_x, len_y) # altitude alt_data = utils.expand_1d_to_2d_array(alt_data, len_y, axis=0) alt_metadata = Metadata(name=alt_name, standard_name=alt_name, shape=new_shape) alt_coord = Coord(alt_data, alt_metadata) # pressure if self.include_pressure: pres_data = hdf.read_data(sdata['Pressure'], self._get_calipso_data) pres_metadata = hdf.read_metadata(sdata['Pressure'], "SD") # Fix badly formatted units which aren't CF compliant and will break if they are aggregated if str(pres_metadata.units) == "hPA": pres_metadata.units = "hPa" pres_metadata.shape = new_shape pres_coord = Coord(pres_data, pres_metadata, 'P') # latitude lat_data = utils.expand_1d_to_2d_array(lat_data[:, index_offset], len_x, axis=1) lat_metadata = hdf.read_metadata(sdata['Latitude'], "SD") lat_metadata.shape = new_shape lat_coord = Coord(lat_data, lat_metadata, 'Y') # longitude lon = sdata['Longitude'] lon_data = hdf.read_data(lon, self._get_calipso_data) lon_data = utils.expand_1d_to_2d_array(lon_data[:, index_offset], len_x, axis=1) lon_metadata = hdf.read_metadata(lon, "SD") lon_metadata.shape = new_shape lon_coord = Coord(lon_data, lon_metadata, 'X') # profile time, x time = sdata['Profile_Time'] time_data = hdf.read_data(time, self._get_calipso_data) time_data = convert_sec_since_to_std_time(time_data, dt.datetime(1993, 1, 1, 0, 0, 0)) time_data = utils.expand_1d_to_2d_array(time_data[:, index_offset], len_x, axis=1) time_coord = Coord(time_data, Metadata(name='Profile_Time', standard_name='time', shape=time_data.shape, units=cis_standard_time_unit), "T") # create the object containing all coordinates coords = CoordList() coords.append(lat_coord) coords.append(lon_coord) coords.append(time_coord) coords.append(alt_coord) if self.include_pressure and (pres_data.shape == alt_data.shape): # For MODIS L1 this may is not be true, so skips the air pressure reading. If required for MODIS L1 then # some kind of interpolation of the air pressure would be required, as it is on a different (smaller) grid # than for the Lidar_Data_Altitudes. coords.append(pres_coord) return coords