def get_variable_names(self, filenames, data_type=None):
"""
Get a list of available variable names from the filenames list passed in. This general implementation can be
overridden in specific products to include/exclude variables which may or may not be relevant.
The data_type parameter can be used to specify extra information.
:param list filenames: List of string filenames of files to be read from
:param str data_type: 'SD' or 'VD' to specify only return SD or VD variables from HDF files. This may take on
other values in specific product implementations.
:return: A set of variable names as strings
:rtype: str
"""
variables = []
for filename in filenames:
file_variables = None
if filename.endswith(".nc"):
file_variables = get_netcdf_file_variables(filename)
remove_variables_with_non_spatiotemporal_dimensions(file_variables, self.valid_dimensions)
elif filename.endswith(".hdf"):
if data_type is None:
data_type = "SD"
sd_vars, vd_vars = get_hdf4_file_variables(filename, data_type)
file_variables = list((sd_vars or {}).keys()) + list((vd_vars or {}).keys())
variables.extend(file_variables)
return set(variables)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
def get_axis_std_name(var):
axis=None
lvar = var.lower()
if lvar.startswith('lon'):
axis = 'x', 'longitude'
if lvar.startswith('lat'):
axis = 'y', 'latitude'
if lvar == 'G_ALT' or lvar == 'altitude' or lvar == 'pressure_altitude':
axis = 'z', 'altitude'
if lvar == 'time':
axis = 't', 'time'
if lvar == 'p' or lvar == 'pressure' or lvar == 'static_pressure':
axis = 'p', 'air_pressure'
return axis
all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None]
# Get rid of any duplicates
coord_variables = []
for v in all_coord_variables:
if v is None or v[1][1] not in [x[1][1] for x in coord_variables]:
coord_variables.append(v)
all_variables = coord_variables.copy()
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis_std_name in coord_variables:
try:
meta = get_metadata(var_data[name][0])
if meta.standard_name is None:
meta.standard_name = axis_std_name[1]
coord = Coord(var_data[name], meta, axis=axis_std_name[0])
if meta.standard_name == 'time':
# Converting units to CIS std time
coord.convert_to_std_time()
coords.append(coord)
except InvalidVariableError:
pass
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
def get_axis_std_name(var):
axis=None
lvar = var.lower()
if lvar == 'longitude':
axis = 'x', 'longitude'
if lvar == 'latitude':
axis = 'y', 'latitude'
if lvar == 'G_ALT' or lvar == 'altitude' or lvar == 'pressure_altitude':
axis = 'z', 'altitude'
if lvar == 'time':
axis = 't', 'time'
if lvar == 'p' or lvar == 'pressure' or lvar == 'static_pressure':
axis = 'p', 'air_pressure'
return axis
all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None]
# Get rid of any duplicates
coord_variables = []
for v in all_coord_variables:
if v is None or v[1][1] not in [x[1][1] for x in coord_variables]:
coord_variables.append(v)
all_variables = coord_variables.copy()
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis_std_name in coord_variables:
try:
meta = get_metadata(var_data[name][0])
if meta.standard_name is None:
meta.standard_name = axis_std_name[1]
coords.append(Coord(var_data[name], meta, axis=axis_std_name[0]))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
def get_axis_std_name(lvar):
axis=None
if lvar == 'LON_JAVAD' or lvar == 'LON_OXTS':
axis = 'x', 'longitude'
if lvar == 'LAT_JAVAD' or lvar == 'LAT_OXTS':
axis = 'y', 'latitude'
if lvar == 'ALT_JAVAD' or lvar == 'ALT_OXTS':
axis = 'z', 'altitude'
if lvar == 'Time':
axis = 't', 'time'
if lvar == 'PS_AIR':
axis = 'p', 'air_pressure'
return axis
all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None]
# Get rid of any duplicates
coord_variables = []
for v in all_coord_variables:
if v is None or v[1][1] not in [x[1][1] for x in coord_variables]:
coord_variables.append(v)
all_variables = coord_variables.copy()
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis_std_name in coord_variables:
try:
meta = get_metadata(var_data[name][0])
if meta.standard_name is None:
meta.standard_name = axis_std_name[1]
coords.append(Coord(var_data[name], meta, axis=axis_std_name[0]))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
def get_variable_names(self, filenames, data_type=None):
from cis.data_io.netcdf import get_netcdf_file_variables, remove_variables_with_non_spatiotemporal_dimensions
variables = []
for filename in filenames:
file_variables = get_netcdf_file_variables(filename)
remove_variables_with_non_spatiotemporal_dimensions(
file_variables, self.valid_dimensions)
variables.extend(file_variables)
return set(variables)
def _load_data_definition(self, filenames):
"""
Load the definition of the data
:param filenames: filenames from which to load the data
:return: variable selector containing the data definitions
"""
variables_list = [get_netcdf_file_variables(f) for f in filenames]
attributes = [get_netcdf_file_attributes(f) for f in filenames]
variable_selector = self.variableSelectorClass(attributes, variables_list)
return variable_selector
def _load_data_definition(self, filenames):
"""
Load the definition of the data
:param filenames: filenames from which to load the data
:return: variable selector containing the data definitions
"""
variables_list = [get_netcdf_file_variables(f) for f in filenames]
attributes = [get_netcdf_file_attributes(f) for f in filenames]
variable_selector = self.variableSelectorClass(attributes, variables_list)
return variable_selector
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
axis_lookup = {
"longitude": "x",
'latitude': 'y',
'altitude': 'z',
'time': 't',
'air_pressure': 'p'
}
coord_variables = [(v, axis_lookup[v]) for v in file_variables
if v in axis_lookup]
# Create a copy to contain all the variables to read
all_variables = list(coord_variables)
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames,
[v[0] for v in all_variables])
for name, axis in coord_variables:
try:
coords.append(
Coord(var_data[name],
get_metadata(var_data[name][0]),
axis=axis))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable],
get_metadata(var_data[usr_variable][0]),
coords)
return res
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
axis_lookup = {"longitude": "x", 'latitude': 'y', 'altitude': 'z', 'time': 't', 'air_pressure': 'p'}
coord_variables = [(v, axis_lookup[v]) for v in file_variables if v in axis_lookup]
# Create a copy to contain all the variables to read
all_variables = list(coord_variables)
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis in coord_variables:
try:
coords.append(Coord(var_data[name], get_metadata(var_data[name][0]), axis=axis))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
