def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord
from cis.exceptions import InvalidVariableError
variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"), ("time", "t"), ("air_pressure", "p")]
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
for variable in variables:
try:
var_data = read_many_files_individually(filenames, variable[0])[variable[0]]
coords.append(Coord(var_data, get_metadata(var_data[0]), axis=variable[1]))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
usr_var_data = read_many_files_individually(filenames, usr_variable)[usr_variable]
res = UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords)
return res
def create_coords(self, filenames, variable=None):
"""
Reads the coordinates and data if required from the files
:param filenames: List of filenames to read coordinates from
:param variable: load a variable for the data
:return: Coordinates
"""
from iris.cube import Cube
from iris.coords import DimCoord
from cis.data_io.netcdf import read
from cis.utils import concatenate
data_variables, variable_selector = self._load_data(filenames, variable)
aux_coords = self._create_coordinates_list(data_variables, variable_selector)
dim_coords = [(DimCoord(np.arange(len(aux_coords[0].points)), var_name='obs'), (0,))]
if variable is None:
raise ValueError("Must specify variable")
aux_coord_name = variable_selector.find_auxiliary_coordinate(variable)
if aux_coord_name is not None:
# We assume that the auxilliary coordinate is the same shape across files
v = read(filenames[0], [aux_coord_name])[aux_coord_name]
aux_meta = get_metadata(v)
# We have to assume the shape here...
dim_coords.append((DimCoord(v[:], var_name=aux_coord_name, units=aux_meta.units,
long_name=aux_meta.long_name), (1,)))
cube_meta = get_metadata(data_variables[variable][0])
return Cube(concatenate([d[:] for d in data_variables[variable]]),
units=cube_meta.units, var_name=variable, long_name=cube_meta.long_name,
dim_coords_and_dims=dim_coords, aux_coords_and_dims=[(c, (0,)) for c in aux_coords])
def create_coords(self, filenames, variable=None):
"""
Reads the coordinates and data if required from the files
:param filenames: List of filenames to read coordinates from
:param variable: load a variable for the data
:return: Coordinates
"""
from cis.data_io.netcdf import read_many_files_individually
from cis.data_io.Coord import Coord, CoordList
from cis.exceptions import InvalidVariableError
variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"), ("time", "t"), ("air_pressure", "p")]
dim_coords = CoordList()
for v in variables:
try:
var_data = read_many_files_individually(filenames, v[0])[v[0]]
dim_coords.append(Coord(var_data, get_metadata(var_data[0]), axis=v[1]))
except InvalidVariableError:
pass
if variable is None:
return UngriddedCoordinates(dim_coords)
else:
all_coords = self._add_aux_coordinate(dim_coords, filenames[0], 'DP_MID',
dim_coords.get_coord(standard_name='time').data.size)
usr_var_data = read_many_files_individually(filenames, variable)[variable]
return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), all_coords)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
variables = [("longitude", "x"), ("latitude", "y")]
# if usr_variable is not None:
# variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
var_data = read_many_files_individually(filenames,
[v[0] for v in variables])
for var, (name, axis) in zip(var_data.values(), variables):
try:
coords.append(Coord(var, get_metadata(var[0]), axis=axis))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
usr_var_data = read_many_files_individually(
filenames, usr_variable)[usr_variable]
res = UngriddedData(usr_var_data, get_metadata(usr_var_data[0]),
coords)
return res
def _create_coord_list(self, filename):
import numpy as np
coords = CoordList()
time_data = read(filename, 'time')['time']
len_x = time_data.shape[0]
try:
alt_data = read(filename, 'altitude')['altitude']
except InvalidVariableError:
alt_data = read(filename, 'range')['range']
len_y = alt_data.shape[0]
time_arr = utils.expand_1d_to_2d_array(time_data[:], len_y, axis=1)
t_coord = Coord(time_arr, get_metadata(time_data), axis='x')
t_coord.convert_to_std_time()
coords.append(t_coord)
alt_arr = utils.expand_1d_to_2d_array(alt_data[:], len_x, axis=0)
coords.append(Coord(alt_arr, get_metadata(alt_data), axis='y'))
lat_data = read(filename, 'latitude')['latitude']
lat_arr = np.ones(alt_arr.shape) * lat_data[:]
coords.append(Coord(lat_arr, get_metadata(lat_data)))
lon_data = read(filename, 'longitude')['longitude']
lon_arr = np.ones(alt_arr.shape) * lon_data[:]
coords.append(Coord(lon_arr, get_metadata(lon_data)))
return coords
def _create_coord_list(self, filenames):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord
from cis.exceptions import InvalidVariableError
try:
variables = ["lon", "lat", "time"]
data = read_many_files_individually(filenames, variables)
except InvalidVariableError:
variables = ["longitude", "latitude", "time"]
data = read_many_files_individually(filenames, variables)
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
coords.append(
Coord(data[variables[0]], get_metadata(data[variables[0]][0]),
"X"))
coords.append(
Coord(data[variables[1]], get_metadata(data[variables[1]][0]),
"Y"))
coords.append(
self._fix_time(
Coord(data[variables[2]], get_metadata(data[variables[2]][0]),
"T")))
return coords
def _create_coord_list(self, filename):
import numpy as np
coords = CoordList()
time_data = read(filename, 'time')['time']
try:
alt_data = read(filename, 'altitude')['altitude']
except InvalidVariableError:
alt_data = read(filename, 'range')['range']
len_y = alt_data.shape[1]
time_arr = utils.expand_1d_to_2d_array(time_data[:], len_y, axis=1)
t_coord = Coord(time_arr, get_metadata(time_data), axis='x')
t_coord.convert_to_std_time()
coords.append(t_coord)
#alt_arr = utils.expand_1d_to_2d_array(alt_data[:], len_x, axis=0)
alt_arr = alt_data[:, :, 0] #eliminate "angle" axis
#alt_arr = alt_data #eliminate "angle" axis
coords.append(Coord(alt_arr, get_metadata(alt_data), axis='y'))
lat_data = read(filename, 'latitude')['latitude']
lat_arr = np.ones(alt_arr.shape) * lat_data[:]
coords.append(Coord(lat_arr, get_metadata(lat_data)))
lon_data = read(filename, 'longitude')['longitude']
lon_arr = np.ones(alt_arr.shape) * lon_data[:]
coords.append(Coord(lon_arr, get_metadata(lon_data)))
return coords
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
variables = [("lon", "x"), ("lat", "y")]
# if usr_variable is not None:
# variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in variables])
for var, (name, axis) in zip(var_data.values(), variables):
try:
coords.append(Coord(var, get_metadata(var[0]), axis=axis))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
usr_var_data = read_many_files_individually(filenames, usr_variable)[usr_variable]
res = UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords)
return res
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
def get_axis_std_name(var):
axis=None
lvar = var.lower()
if lvar.startswith('lon'):
axis = 'x', 'longitude'
if lvar.startswith('lat'):
axis = 'y', 'latitude'
if lvar == 'G_ALT' or lvar == 'altitude' or lvar == 'pressure_altitude':
axis = 'z', 'altitude'
if lvar == 'time':
axis = 't', 'time'
if lvar == 'p' or lvar == 'pressure' or lvar == 'static_pressure':
axis = 'p', 'air_pressure'
return axis
all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None]
# Get rid of any duplicates
coord_variables = []
for v in all_coord_variables:
if v is None or v[1][1] not in [x[1][1] for x in coord_variables]:
coord_variables.append(v)
all_variables = coord_variables.copy()
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis_std_name in coord_variables:
try:
meta = get_metadata(var_data[name][0])
if meta.standard_name is None:
meta.standard_name = axis_std_name[1]
coord = Coord(var_data[name], meta, axis=axis_std_name[0])
if meta.standard_name == 'time':
# Converting units to CIS std time
coord.convert_to_std_time()
coords.append(coord)
except InvalidVariableError:
pass
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
def get_axis_std_name(var):
axis=None
lvar = var.lower()
if lvar == 'longitude':
axis = 'x', 'longitude'
if lvar == 'latitude':
axis = 'y', 'latitude'
if lvar == 'G_ALT' or lvar == 'altitude' or lvar == 'pressure_altitude':
axis = 'z', 'altitude'
if lvar == 'time':
axis = 't', 'time'
if lvar == 'p' or lvar == 'pressure' or lvar == 'static_pressure':
axis = 'p', 'air_pressure'
return axis
all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None]
# Get rid of any duplicates
coord_variables = []
for v in all_coord_variables:
if v is None or v[1][1] not in [x[1][1] for x in coord_variables]:
coord_variables.append(v)
all_variables = coord_variables.copy()
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis_std_name in coord_variables:
try:
meta = get_metadata(var_data[name][0])
if meta.standard_name is None:
meta.standard_name = axis_std_name[1]
coords.append(Coord(var_data[name], meta, axis=axis_std_name[0]))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
def get_axis_std_name(lvar):
axis=None
if lvar == 'LON_JAVAD' or lvar == 'LON_OXTS':
axis = 'x', 'longitude'
if lvar == 'LAT_JAVAD' or lvar == 'LAT_OXTS':
axis = 'y', 'latitude'
if lvar == 'ALT_JAVAD' or lvar == 'ALT_OXTS':
axis = 'z', 'altitude'
if lvar == 'Time':
axis = 't', 'time'
if lvar == 'PS_AIR':
axis = 'p', 'air_pressure'
return axis
all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None]
# Get rid of any duplicates
coord_variables = []
for v in all_coord_variables:
if v is None or v[1][1] not in [x[1][1] for x in coord_variables]:
coord_variables.append(v)
all_variables = coord_variables.copy()
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis_std_name in coord_variables:
try:
meta = get_metadata(var_data[name][0])
if meta.standard_name is None:
meta.standard_name = axis_std_name[1]
coords.append(Coord(var_data[name], meta, axis=axis_std_name[0]))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
def create_coords(self, filenames):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates
var_data = read_many_files_individually(filenames, ["longitude", "latitude", "time"])
lon = Coord(var_data["longitude"], get_metadata(var_data["longitude"][0]), axis="x")
lat = Coord(var_data["latitude"], get_metadata(var_data["latitude"][0]), axis="y")
time = Coord(var_data["time"], get_metadata(var_data["time"][0]), axis="t")
coords = CoordList([lat, lon, time])
return UngriddedCoordinates(coords)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
axis_lookup = {
"longitude": "x",
'latitude': 'y',
'altitude': 'z',
'time': 't',
'air_pressure': 'p'
}
coord_variables = [(v, axis_lookup[v]) for v in file_variables
if v in axis_lookup]
# Create a copy to contain all the variables to read
all_variables = list(coord_variables)
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames,
[v[0] for v in all_variables])
for name, axis in coord_variables:
try:
coords.append(
Coord(var_data[name],
get_metadata(var_data[name][0]),
axis=axis))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable],
get_metadata(var_data[usr_variable][0]),
coords)
return res
def create_data_object(self, filenames, variable):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedData
var_data = read_many_files_individually(filenames, ["longitude", "latitude", "time", variable])
lon = Coord(var_data["longitude"], get_metadata(var_data["longitude"][0]), axis="x")
lat = Coord(var_data["latitude"], get_metadata(var_data["latitude"][0]), axis="y")
time = Coord(var_data["time"], get_metadata(var_data["time"][0]), axis="t")
coords = CoordList([lat, lon, time])
usr_var_data = var_data[variable]
return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords)
def create_coords(self, filenames, variable=None):
"""
Reads the coordinates and data if required from the files
:param filenames: List of filenames to read coordinates from
:param variable: load a variable for the data
:return: Coordinates
"""
data_variables, variable_selector = self._load_data(
filenames, variable)
dim_coords = self._create_coordinates_list(data_variables,
variable_selector)
if variable is None:
return UngriddedCoordinates(dim_coords)
else:
aux_coord_name = variable_selector.find_auxiliary_coordinate(
variable)
if aux_coord_name is not None:
all_coords = self._add_aux_coordinate(
dim_coords, filenames[0], aux_coord_name,
dim_coords.get_coord(standard_name='time').data.size)
else:
all_coords = dim_coords
return UngriddedData(data_variables[variable],
get_metadata(data_variables[variable][0]),
all_coords)
def _create_time_coord(self, timestamp, time_variable_name, data_variables, coord_axis='T', standard_name='time'):
"""
Create a time coordinate, taking into account the fact that each file may have a different timestamp.
:param timestamp: Timestamp or list of timestamps for
:param time_variable_name: Name of the time variable
:param data_variables: Dictionary containing one or multiple netCDF data variables for each variable name
:param coord_axis: Axis, default 'T'
:param standard_name: Coord standard name, default 'time'
:return: Coordinate
"""
from cis.data_io.Coord import Coord
from six.moves import zip_longest
timestamps = listify(timestamp)
time_variables = data_variables[time_variable_name]
time_coords = []
# Create a coordinate for each separate file to account for differing timestamps
for file_time_var, timestamp in zip_longest(time_variables, timestamps):
metadata = get_metadata(file_time_var)
metadata.standard_name = standard_name
coord = Coord(file_time_var, metadata, coord_axis)
coord.convert_to_std_time(timestamp)
time_coords.append(coord)
return Coord.from_many_coordinates(time_coords)
def _add_aux_coordinate(dim_coords, filename, aux_coord_name, length):
"""
Add an auxiliary coordinate to a list of (reshaped) dimension coordinates
:param dim_coords: CoordList of one-dimensional coordinates representing physical dimensions
:param filename: The data file containing the aux coord
:param aux_coord_name: The name of the aux coord to add to the coord list
:param length: The length of the data dimensions which this auxiliary coordinate should span
:return: A CoordList of reshaped (2D) physical coordinates plus the 2D auxiliary coordinate
"""
from cis.data_io.Coord import Coord
from cis.utils import expand_1d_to_2d_array
from cis.data_io.netcdf import read
# We assume that the auxilliary coordinate is the same shape across files
d = read(filename, [aux_coord_name])[aux_coord_name]
# Reshape to the length given
aux_data = expand_1d_to_2d_array(d[:], length, axis=0)
# Get the length of the auxiliary coordinate
len_y = d[:].size
for dim_coord in dim_coords:
dim_coord.data = expand_1d_to_2d_array(dim_coord.data, len_y, axis=1)
all_coords = dim_coords + [Coord(aux_data, get_metadata(d))]
return all_coords
def _create_time_coord(self, timestamp, time_variable_name, data_variables, coord_axis='T', standard_name='time'):
"""
Create a time coordinate, taking into account the fact that each file may have a different timestamp.
:param timestamp: Timestamp or list of timestamps for
:param time_variable_name: Name of the time variable
:param data_variables: Dictionary containing one or multiple netCDF data variables for each variable name
:param coord_axis: Axis, default 'T'
:param standard_name: Coord standard name, default 'time'
:return: Coordinate
"""
from iris.coords import AuxCoord
from six.moves import zip_longest
from cis.time_util import convert_time_using_time_stamp_info_to_std_time as convert, cis_standard_time_unit
from cis.utils import concatenate
timestamps = listify(timestamp)
time_variables = data_variables[time_variable_name]
time_data = []
# Create a coordinate for each separate file to account for differing timestamps
for file_time_var, timestamp in zip_longest(time_variables, timestamps):
metadata = get_metadata(file_time_var)
if timestamp is not None:
time_d = convert(file_time_var[:], metadata.units, timestamp)
else:
time_d = metadata.units.convert(file_time_var[:], cis_standard_time_unit)
time_data.append(time_d)
return AuxCoord(concatenate(time_data), standard_name=standard_name, units=cis_standard_time_unit)
def create_data_object(self, filenames, variable):
from cis.data_io.netcdf import get_metadata, read_many_files_individually
coords = self._create_coord_list(filenames)
var = read_many_files_individually(filenames, [variable])
metadata = get_metadata(var[variable][0])
return UngriddedData(var[variable], metadata, coords)
def create_data_object(self, filenames, variable):
from cis.data_io.netcdf import get_metadata, read_many_files_individually
coords = self._create_coord_list(filenames)
var = read_many_files_individually(filenames, [variable])
metadata = get_metadata(var[variable][0])
return UngriddedData(var[variable], metadata, coords)
def create_data_object(self, filenames, variable):
from cis.data_io.netcdf import read_many_files, get_metadata
coords = self._create_coord_list(filenames)
data = read_many_files(filenames, variable, dim="pixel_number")
metadata = get_metadata(data[variable])
return UngriddedData(data[variable], metadata, coords)
def create_data_object(self, filenames, variable):
from cis.data_io.ungridded_data import UngriddedData
usr_var_data = read_many_files_individually(filenames,
variable)[variable]
coords = self.create_coords(filename)
return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]),
coords)
def create_data_object(self, filenames, variable):
logging.debug("Creating data object for variable " + variable)
# reading coordinates
# the variable here is needed to work out whether to apply interpolation to the lat/lon data or not
coords = self._create_coord_list(filenames[0])
usr_var_data = read_many_files_individually(filenames, variable)[variable]
return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.data_io.Coord import Coord, CoordList
from cis.exceptions import InvalidVariableError
variables = [("lon", "x", 'longitude'), ("lat", "y", 'latitude'),
("alt", "z", 'altitude'), ("time", "t", 'time'),
("p", "p", 'air_pressure')]
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
for variable in variables:
try:
var_data = read_many_files_individually(
filenames, variable[0])[variable[0]]
meta = get_metadata(var_data[0])
meta.standard_name = variable[2]
# Some of the variables have an illegal name attribute...
meta.misc.pop('name', None)
c = Coord(var_data, meta, axis=variable[1])
if variable[1] == 'z':
c.convert_units('m')
coords.append(c)
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
usr_var_data = read_many_files_individually(
filenames, usr_variable)[usr_variable]
meta = get_metadata(usr_var_data[0])
# Some of the variables have an illegal name attribute...
meta.misc.pop('name', None)
res = UngriddedData(usr_var_data, meta, coords)
return res
def _create_coord_list(self, filenames):
from cis.data_io.netcdf import read_many_files, get_metadata
from cis.data_io.Coord import Coord
import datetime
# FIXME: when reading an existing file variables might be "latitude", "longitude"
variables = ["lat", "lon", "time"]
logging.info("Listing coordinates: " + str(variables))
data = read_many_files(filenames, variables, dim="pixel_number")
coords = CoordList()
coords.append(Coord(data["lon"], get_metadata(data["lon"]), "X"))
coords.append(Coord(data["lat"], get_metadata(data["lat"]), "Y"))
time_coord = Coord(data["time"], get_metadata(data["time"]), "T")
time_coord.convert_TAI_time_to_std_time(datetime.datetime(1970, 1, 1))
coords.append(time_coord)
return coords
def _create_coord_list(self, filenames):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord
variables = ["lat", "lon", "time"]
logging.info("Listing coordinates: " + str(variables))
var_data = read_many_files_individually(filenames, variables)
coords = CoordList()
coords.append(Coord(var_data['lat'], get_metadata(var_data['lat'][0]), 'Y'))
coords.append(Coord(var_data['lon'], get_metadata(var_data['lon'][0]), 'X'))
time_coord = Coord(var_data['time'], get_metadata(var_data['time'][0]))
# TODO: Is this really julian?
time_coord.convert_julian_to_std_time()
coords.append(time_coord)
return coords
def create_coords(self, filenames, variable=None):
"""
Reads the coordinates and data if required from the files
:param filenames: List of filenames to read coordinates from
:param variable: load a variable for the data
:return: Coordinates
"""
from iris.cube import Cube
from iris.coords import DimCoord
from cis.data_io.netcdf import read
from cis.utils import concatenate
data_variables, variable_selector = self._load_data(
filenames, variable)
aux_coords = self._create_coordinates_list(data_variables,
variable_selector)
dim_coords = [(DimCoord(np.arange(len(aux_coords[0].points)),
var_name='obs'), (0, ))]
if variable is None:
raise ValueError("Must specify variable")
aux_coord_name = variable_selector.find_auxiliary_coordinate(variable)
if aux_coord_name is not None:
# We assume that the auxilliary coordinate is the same shape across files
v = read(filenames[0], [aux_coord_name])[aux_coord_name]
aux_meta = get_metadata(v)
# We have to assume the shape here...
dim_coords.append((DimCoord(v[:],
var_name=aux_coord_name,
units=aux_meta.units,
long_name=aux_meta.long_name), (1, )))
cube_meta = get_metadata(data_variables[variable][0])
return Cube(concatenate([d[:] for d in data_variables[variable]]),
units=cube_meta.units,
var_name=variable,
long_name=cube_meta.long_name,
dim_coords_and_dims=dim_coords,
aux_coords_and_dims=[(c, (0, )) for c in aux_coords])
def _create_coord_list(self, filenames):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord
from cis.exceptions import InvalidVariableError
try:
variables = ["lon", "lat", "time"]
data = read_many_files_individually(filenames, variables)
except InvalidVariableError:
variables = ["longitude", "latitude", "time"]
data = read_many_files_individually(filenames, variables)
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
coords.append(Coord(data[variables[0]], get_metadata(data[variables[0]][0]), "X"))
coords.append(Coord(data[variables[1]], get_metadata(data[variables[1]][0]), "Y"))
coords.append(self._fix_time(Coord(data[variables[2]], get_metadata(data[variables[2]][0]), "T")))
return coords
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
axis_lookup = {"longitude": "x", 'latitude': 'y', 'altitude': 'z', 'time': 't', 'air_pressure': 'p'}
coord_variables = [(v, axis_lookup[v]) for v in file_variables if v in axis_lookup]
# Create a copy to contain all the variables to read
all_variables = list(coord_variables)
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis in coord_variables:
try:
coords.append(Coord(var_data[name], get_metadata(var_data[name][0]), axis=axis))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
def create_coords(self, filenames, variable=None):
"""
Reads the coordinates and data if required from the files
:param filenames: List of filenames to read coordinates from
:param variable: load a variable for the data
:return: Coordinates
"""
data_variables, variable_selector = self._load_data(filenames, variable)
coords = self._create_coordinates_list(data_variables, variable_selector)
if variable is None:
return UngriddedCoordinates(coords)
else:
return UngriddedData(data_variables[variable], get_metadata(data_variables[variable][0]), coords)
def _create_coord(self, coord_axis, data_variable_name, data_variables, standard_name):
"""
Create a coordinate for the co-ordinate list
:param coord_axis: axis of the coordinate in the coords
:param data_variable_name: the name of the variable in the data
:param data_variables: the data variables
:param standard_name: the standard name it should have
:return: a coords object
"""
from iris.coords import AuxCoord
from cis.utils import concatenate
data = concatenate([d[:] for d in data_variables[data_variable_name]])
m = get_metadata(data_variables[data_variable_name][0])
return AuxCoord(data, units=m.units, standard_name=standard_name)
def _create_coord(self, coord_axis, data_variable_name, data_variables,
standard_name):
"""
Create a coordinate for the co-ordinate list
:param coord_axis: axis of the coordinate in the coords
:param data_variable_name: the name of the variable in the data
:param data_variables: the data variables
:param standard_name: the standard name it should have
:return: a coords object
"""
from iris.coords import AuxCoord
from cis.utils import concatenate
data = concatenate([d[:] for d in data_variables[data_variable_name]])
m = get_metadata(data_variables[data_variable_name][0])
return AuxCoord(data, units=m.units, standard_name=standard_name)
def _create_coord(self, coord_axis, data_variable_name, data_variables, standard_name):
"""
Create a coordinate for the co-ordinate list
:param coord_axis: axis of the coordinate in the coords
:param data_variable_name: the name of the variable in the data
:param data_variables: the data variables
:param standard_name: the standard name it should have
:return: a coords object
"""
from cis.data_io.Coord import Coord
coordinate_data_objects = []
for d in data_variables[data_variable_name]:
m = get_metadata(d)
m.standard_name = standard_name
coordinate_data_objects.append(Coord(d, m, coord_axis))
return Coord.from_many_coordinates(coordinate_data_objects)
def _create_coord(self, coord_axis, data_variable_name, data_variables, standard_name):
"""
Create a coordinate for the co-ordinate list
:param coord_axis: axis of the coordinate in the coords
:param data_variable_name: the name of the variable in the data
:param data_variables: the data variables
:param standard_name: the standard name it should have
:return: a coords object
"""
from cis.data_io.Coord import Coord
coordinate_data_objects = []
for d in data_variables[data_variable_name]:
m = get_metadata(d)
m.alter_standard_name(standard_name)
coordinate_data_objects.append(Coord(d, m, coord_axis))
return Coord.from_many_coordinates(coordinate_data_objects)
def create_data_object(self, filenames, variable):
logging.debug("Creating data object for variable " + variable)
# reading coordinates
# the variable here is needed to work out whether to apply interpolation to the lat/lon data or not
coords = self._create_coord_list(filenames[0])
usr_var_data = read_many_files_individually(filenames,
variable)[variable]
qc_flag = read_many_files_individually(filenames, "qc_flag")["qc_flag"]
import numpy as np
mask = (qc_flag[0][:, :, 0] != 1)
#retdata = np.ma.array(usr_var_data[0][:, :, 0],mask = mask)
from cis.utils import apply_mask_to_numpy_array, concatenate
retdata = apply_mask_to_numpy_array(usr_var_data[0][:, :, 0], mask)
return UngriddedData(retdata, get_metadata(usr_var_data[0]), coords)
def _create_coord(self, coord_axis, data_variable_name, data_variables,
standard_name):
"""
Create a coordinate for the co-ordinate list
:param coord_axis: axis of the coordinate in the coords
:param data_variable_name: the name of the variable in the data
:param data_variables: the data variables
:param standard_name: the standard name it should have
:return: a coords object
"""
from cis.data_io.netcdf import get_metadata
from iris.coords import AuxCoord
from cis.utils import concatenate
from cf_units import Unit
import logging
data = concatenate(
[get_data(d) for d in data_variables[data_variable_name]])
m = get_metadata(data_variables[data_variable_name][0])
m._name = m._name.lower()
m.standard_name = standard_name
if standard_name == 'air_pressure':
if not isinstance(m.units, Unit):
if ',' in m.units:
# Try splitting any commas out
m.units = m.units.split(',')[0]
if ' ' in m.units:
# Try splitting any spaces out
m.units = m.units.split()[0]
if str(m.units) == 'mb' or str(m.units) == 'Mb':
# Try converting to standard nomencleture
m.units = 'mbar'
if str(m.units) == 'hpa':
m.units = 'hPa'
logging.info("Parsed air pressure units {old}".format(old=m.units))
logging.info('Converting to hPa')
if not isinstance(m.units, str):
data = m.units.convert(data, 'hPa')
m.units = 'hPa'
return AuxCoord(data, units=m.units, standard_name=standard_name)
def _create_coord(self, coord_axis, data_variable_name, data_variables, standard_name):
"""
Create a coordinate for the co-ordinate list
:param coord_axis: axis of the coordinate in the coords
:param data_variable_name: the name of the variable in the data
:param data_variables: the data variables
:param standard_name: the standard name it should have
:return: a coords object
"""
from cis.data_io.netcdf import get_metadata
from cis.data_io.Coord import Coord
from cf_units import Unit
import logging
coordinate_data_objects = []
for d in data_variables[data_variable_name]:
data = get_data(d)
m = get_metadata(d)
m._name = m._name.lower()
m.standard_name = standard_name
if standard_name == 'air_pressure':
if not isinstance(m.units, Unit):
if ',' in m.units:
# Try splitting any commas out
m.units = m.units.split(',')[0]
if ' ' in m.units:
# Try splitting any spaces out
m.units = m.units.split()[0]
if str(m.units) == 'mb' or str(m.units) == 'Mb':
# Try converting to standard nomencleture
m.units = 'mbar'
if str(m.units) == 'hpa':
m.units = 'hPa'
logging.info("Parsed air pressure units {old}".format(old=m.units))
logging.info('Converting to hPa')
if not isinstance(m.units, str):
data = m.units.convert(data, 'hPa')
m.units = 'hPa'
coordinate_data_objects.append(Coord(data, m, coord_axis))
return Coord.from_many_coordinates(coordinate_data_objects)
def create_coords(self, filenames, variable=None):
"""
Reads the coordinates and data if required from the files
:param filenames: List of filenames to read coordinates from
:param variable: load a variable for the data
:return: Coordinates
"""
data_variables, variable_selector = self._load_data(filenames, variable)
dim_coords = self._create_coordinates_list(data_variables, variable_selector)
if variable is None:
return UngriddedCoordinates(dim_coords)
else:
aux_coord_name = variable_selector.find_auxiliary_coordinate(variable)
if aux_coord_name is not None:
all_coords = self._add_aux_coordinate(dim_coords, filenames[0], aux_coord_name,
dim_coords.get_coord(standard_name='time').data.size)
else:
all_coords = dim_coords
return UngriddedData(data_variables[variable], get_metadata(data_variables[variable][0]), all_coords)
def _create_time_coord(self,
timestamp,
time_variable_name,
data_variables,
coord_axis='T',
standard_name='time'):
"""
Create a time coordinate, taking into account the fact that each file may have a different timestamp.
:param timestamp: Timestamp or list of timestamps for
:param time_variable_name: Name of the time variable
:param data_variables: Dictionary containing one or multiple netCDF data variables for each variable name
:param coord_axis: Axis, default 'T'
:param standard_name: Coord standard name, default 'time'
:return: Coordinate
"""
from iris.coords import AuxCoord
from six.moves import zip_longest
from cis.time_util import convert_time_using_time_stamp_info_to_std_time as convert, cis_standard_time_unit
from cis.utils import concatenate
timestamps = listify(timestamp)
time_variables = data_variables[time_variable_name]
time_data = []
# Create a coordinate for each separate file to account for differing timestamps
for file_time_var, timestamp in zip_longest(time_variables,
timestamps):
metadata = get_metadata(file_time_var)
if timestamp is not None:
time_d = convert(file_time_var[:], metadata.units, timestamp)
else:
time_d = metadata.units.convert(file_time_var[:],
cis_standard_time_unit)
time_data.append(time_d)
return AuxCoord(concatenate(time_data),
standard_name=standard_name,
units=cis_standard_time_unit)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates
from cis.exceptions import InvalidVariableError
variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"),
("time", "t"), ("relative_humidity", "RH"),
("surface_air_pressure", "Pa"), ("air_temprature", "K"),
("wind_speed"), ("Wind Diretion"), ("rainfall_rate")]
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
for variable in variables:
try:
var_data = read_many_files_individually(
filenames, variable[0])[variable[0]]
coords.append(
Coord(var_data,
get_metadata(var_data[0]),
axis=variable[1]))
except InvalidVariableError:
pass
return UngriddedCoordinates(coords)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates
from cis.exceptions import InvalidVariableError
variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"),
("time", "t"),
("aerosol_backscatter_coefficient", "m-1 sr-1")]
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
for variable in variables:
try:
var_data = read_many_files_individually(
filenames, variable[0])[variable[0]]
coords.append(
Coord(var_data,
get_metadata(var_data[0]),
axis=variable[1]))
except InvalidVariableError:
pass
return UngriddedCoordinates(coords)
def create_coords(self, filenames, variable=None):
"""
Override the default read-in to also read in CCN quality flag data and apply the appropriate mask. We have
to do this before creating the UngriddedData object so that the missing coords don't get fixed first
"""
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.utils import apply_mask_to_numpy_array, concatenate
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
data_variables, variable_selector = self._load_data(filenames, variable)
dim_coords = self._create_coordinates_list(data_variables, variable_selector)
if variable is None:
return UngriddedCoordinates(dim_coords)
else:
aux_coord_name = variable_selector.find_auxiliary_coordinate(variable)
if aux_coord_name is not None:
all_coords = self._add_aux_coordinate(dim_coords, filenames[0], aux_coord_name,
dim_coords.get_coord(standard_name='time').data.size)
else:
all_coords = dim_coords
var_data = data_variables[variable]
if variable and variable.startswith('CCN_COL'):
# Work out the associated variable name for this column
ccn_flag_var = "COL{}_FLAG".format(variable[-1])
# Read in the flags
flags = concatenate([get_data(v) for v in read_many_files_individually(filenames, ccn_flag_var)[
ccn_flag_var]])
# 0 and 1 are both OK
mask = flags > 1
# If a variable was supplied then coords must be an ungridded data object, apply the mask to it
var_data = apply_mask_to_numpy_array(concatenate([get_data(v) for v in var_data]), mask)
return UngriddedData(var_data, get_metadata(data_variables[variable][0]), all_coords)
def create_data_object(self, filenames, variable):
logging.debug("Creating data object for variable " + variable)
variables = [("ER2_IMU/Longitude", "x"), ("ER2_IMU/Latitude", "y"),
("ER2_IMU/gps_time", "t"), ("State/Pressure", "p"),
("DataProducts/Altitude", "z"), ("header/date", ""),
(variable, '')]
logging.info("Listing coordinates: " + str(variables))
var_data = read_many_files_individually(filenames,
[v[0] for v in variables])
date_times = []
for times, date in zip(var_data['ER2_IMU/gps_time'],
var_data['header/date']):
# Date is stored as an array (of length 92??) of floats with format: yyyymmdd
date_str = str(int(date[0]))
t_unit = Unit('hours since {}-{}-{} 00:00:00'.format(
date_str[0:4], date_str[4:6], date_str[6:8]))
date_times.append(
t_unit.convert(get_data(times), cis_standard_time_unit))
# time_data = utils.concatenate([get_data(i) for i in var_data['ER2_IMU/gps_time']])
# date_str = str(int(var_data['header/date'][0][0]))
# Flatten the data by taking the 0th column of the transpose
time_coord = DimCoord(utils.concatenate(date_times).T[0],
standard_name='time',
units=cis_standard_time_unit)
# TODO This won't work for multiple files since the altitude bins are different for each flight...
alt_data = utils.concatenate(
[get_data(i) for i in var_data["DataProducts/Altitude"]])
alt_coord = DimCoord(alt_data[0], standard_name='altitude', units='m')
pres_data = utils.concatenate(
[get_data(i) for i in var_data["State/Pressure"]])
pres_coord = AuxCoord(pres_data,
standard_name='air_pressure',
units='atm')
# Fix the air-pressure units
pres_coord.convert_units('hPa')
lat_data = utils.concatenate(
[get_data(i) for i in var_data['ER2_IMU/Latitude']])
lat_coord = AuxCoord(lat_data.T[0], standard_name='latitude')
lon_data = utils.concatenate(
[get_data(i) for i in var_data['ER2_IMU/Longitude']])
lon_coord = AuxCoord(lon_data.T[0], standard_name='longitude')
data = utils.concatenate([get_data(i) for i in var_data[variable]])
metadata = get_metadata(var_data[variable][0])
cube = Cube(np.ma.masked_invalid(data),
long_name=metadata.misc['Description'],
units=self.clean_units(metadata.units),
dim_coords_and_dims=[(alt_coord, 1), (time_coord, 0)],
aux_coords_and_dims=[(lat_coord, (0, )),
(lon_coord, (0, )),
(pres_coord, (0, 1))])
gd = GriddedData.make_from_cube(cube)
return gd
