def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
variables = [("lon", "x"), ("lat", "y")]
# if usr_variable is not None:
# variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in variables])
for var, (name, axis) in zip(var_data.values(), variables):
try:
coords.append(Coord(var, get_metadata(var[0]), axis=axis))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
usr_var_data = read_many_files_individually(filenames, usr_variable)[usr_variable]
res = UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords)
return res
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
variables = [("longitude", "x"), ("latitude", "y")]
# if usr_variable is not None:
# variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
var_data = read_many_files_individually(filenames,
[v[0] for v in variables])
for var, (name, axis) in zip(var_data.values(), variables):
try:
coords.append(Coord(var, get_metadata(var[0]), axis=axis))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
usr_var_data = read_many_files_individually(
filenames, usr_variable)[usr_variable]
res = UngriddedData(usr_var_data, get_metadata(usr_var_data[0]),
coords)
return res
def _create_coord_list(self, filenames):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord
from cis.exceptions import InvalidVariableError
try:
variables = ["lon", "lat", "time"]
data = read_many_files_individually(filenames, variables)
except InvalidVariableError:
variables = ["longitude", "latitude", "time"]
data = read_many_files_individually(filenames, variables)
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
coords.append(
Coord(data[variables[0]], get_metadata(data[variables[0]][0]),
"X"))
coords.append(
Coord(data[variables[1]], get_metadata(data[variables[1]][0]),
"Y"))
coords.append(
self._fix_time(
Coord(data[variables[2]], get_metadata(data[variables[2]][0]),
"T")))
return coords
def create_coords(self, filenames, variable=None):
"""
Reads the coordinates and data if required from the files
:param filenames: List of filenames to read coordinates from
:param variable: load a variable for the data
:return: Coordinates
"""
from cis.data_io.netcdf import read_many_files_individually
from cis.data_io.Coord import Coord, CoordList
from cis.exceptions import InvalidVariableError
variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"), ("time", "t"), ("air_pressure", "p")]
dim_coords = CoordList()
for v in variables:
try:
var_data = read_many_files_individually(filenames, v[0])[v[0]]
dim_coords.append(Coord(var_data, get_metadata(var_data[0]), axis=v[1]))
except InvalidVariableError:
pass
if variable is None:
return UngriddedCoordinates(dim_coords)
else:
all_coords = self._add_aux_coordinate(dim_coords, filenames[0], 'DP_MID',
dim_coords.get_coord(standard_name='time').data.size)
usr_var_data = read_many_files_individually(filenames, variable)[variable]
return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), all_coords)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord
from cis.exceptions import InvalidVariableError
variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"), ("time", "t"), ("air_pressure", "p")]
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
for variable in variables:
try:
var_data = read_many_files_individually(filenames, variable[0])[variable[0]]
coords.append(Coord(var_data, get_metadata(var_data[0]), axis=variable[1]))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
usr_var_data = read_many_files_individually(filenames, usr_variable)[usr_variable]
res = UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords)
return res
def _load_data(self, filenames, variable):
"""
Open the file and find the correct variables to load in
:param filenames: the filenames to load
:param variable: an extra variable to load
:return: a list of load data and the variable selector used to load name it
"""
variable_selector = self._load_data_definition(filenames)
variables_list = [variable_selector.time_variable_name]
add_to_list_if_not_none(variable_selector.latitude_variable_name,
variables_list)
add_to_list_if_not_none(variable_selector.longitude_variable_name,
variables_list)
add_to_list_if_not_none(variable_selector.altitude_variable_name,
variables_list)
add_to_list_if_not_none(variable_selector.pressure_variable_name,
variables_list)
logging.info("Listing coordinates: " + str(variables_list))
add_to_list_if_not_none(variable, variables_list)
data_variables = read_many_files_individually(filenames,
variables_list)
return data_variables, variable_selector
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
def get_axis_std_name(var):
axis=None
lvar = var.lower()
if lvar.startswith('lon'):
axis = 'x', 'longitude'
if lvar.startswith('lat'):
axis = 'y', 'latitude'
if lvar == 'G_ALT' or lvar == 'altitude' or lvar == 'pressure_altitude':
axis = 'z', 'altitude'
if lvar == 'time':
axis = 't', 'time'
if lvar == 'p' or lvar == 'pressure' or lvar == 'static_pressure':
axis = 'p', 'air_pressure'
return axis
all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None]
# Get rid of any duplicates
coord_variables = []
for v in all_coord_variables:
if v is None or v[1][1] not in [x[1][1] for x in coord_variables]:
coord_variables.append(v)
all_variables = coord_variables.copy()
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis_std_name in coord_variables:
try:
meta = get_metadata(var_data[name][0])
if meta.standard_name is None:
meta.standard_name = axis_std_name[1]
coord = Coord(var_data[name], meta, axis=axis_std_name[0])
if meta.standard_name == 'time':
# Converting units to CIS std time
coord.convert_to_std_time()
coords.append(coord)
except InvalidVariableError:
pass
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
def create_data_object(self, filenames, variable):
from cis.data_io.netcdf import get_metadata, read_many_files_individually
coords = self._create_coord_list(filenames)
var = read_many_files_individually(filenames, [variable])
metadata = get_metadata(var[variable][0])
return UngriddedData(var[variable], metadata, coords)
def create_data_object(self, filenames, variable):
from cis.data_io.netcdf import get_metadata, read_many_files_individually
coords = self._create_coord_list(filenames)
var = read_many_files_individually(filenames, [variable])
metadata = get_metadata(var[variable][0])
return UngriddedData(var[variable], metadata, coords)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
def get_axis_std_name(var):
axis=None
lvar = var.lower()
if lvar == 'longitude':
axis = 'x', 'longitude'
if lvar == 'latitude':
axis = 'y', 'latitude'
if lvar == 'G_ALT' or lvar == 'altitude' or lvar == 'pressure_altitude':
axis = 'z', 'altitude'
if lvar == 'time':
axis = 't', 'time'
if lvar == 'p' or lvar == 'pressure' or lvar == 'static_pressure':
axis = 'p', 'air_pressure'
return axis
all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None]
# Get rid of any duplicates
coord_variables = []
for v in all_coord_variables:
if v is None or v[1][1] not in [x[1][1] for x in coord_variables]:
coord_variables.append(v)
all_variables = coord_variables.copy()
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis_std_name in coord_variables:
try:
meta = get_metadata(var_data[name][0])
if meta.standard_name is None:
meta.standard_name = axis_std_name[1]
coords.append(Coord(var_data[name], meta, axis=axis_std_name[0]))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
def create_data_object(self, filenames, variable):
from cis.data_io.ungridded_data import UngriddedData
usr_var_data = read_many_files_individually(filenames,
variable)[variable]
coords = self.create_coords(filename)
return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]),
coords)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
def get_axis_std_name(lvar):
axis=None
if lvar == 'LON_JAVAD' or lvar == 'LON_OXTS':
axis = 'x', 'longitude'
if lvar == 'LAT_JAVAD' or lvar == 'LAT_OXTS':
axis = 'y', 'latitude'
if lvar == 'ALT_JAVAD' or lvar == 'ALT_OXTS':
axis = 'z', 'altitude'
if lvar == 'Time':
axis = 't', 'time'
if lvar == 'PS_AIR':
axis = 'p', 'air_pressure'
return axis
all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None]
# Get rid of any duplicates
coord_variables = []
for v in all_coord_variables:
if v is None or v[1][1] not in [x[1][1] for x in coord_variables]:
coord_variables.append(v)
all_variables = coord_variables.copy()
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis_std_name in coord_variables:
try:
meta = get_metadata(var_data[name][0])
if meta.standard_name is None:
meta.standard_name = axis_std_name[1]
coords.append(Coord(var_data[name], meta, axis=axis_std_name[0]))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
def create_data_object(self, filenames, variable):
logging.debug("Creating data object for variable " + variable)
# reading coordinates
# the variable here is needed to work out whether to apply interpolation to the lat/lon data or not
coords = self._create_coord_list(filenames[0])
usr_var_data = read_many_files_individually(filenames,
variable)[variable]
qc_flag = read_many_files_individually(filenames, "qc_flag")["qc_flag"]
import numpy as np
mask = (qc_flag[0][:, :, 0] != 1)
#retdata = np.ma.array(usr_var_data[0][:, :, 0],mask = mask)
from cis.utils import apply_mask_to_numpy_array, concatenate
retdata = apply_mask_to_numpy_array(usr_var_data[0][:, :, 0], mask)
return UngriddedData(retdata, get_metadata(usr_var_data[0]), coords)
def create_data_object(self, filenames, variable):
logging.debug("Creating data object for variable " + variable)
# reading coordinates
# the variable here is needed to work out whether to apply interpolation to the lat/lon data or not
coords = self._create_coord_list(filenames[0])
usr_var_data = read_many_files_individually(filenames, variable)[variable]
return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.data_io.Coord import Coord, CoordList
from cis.exceptions import InvalidVariableError
variables = [("lon", "x", 'longitude'), ("lat", "y", 'latitude'),
("alt", "z", 'altitude'), ("time", "t", 'time'),
("p", "p", 'air_pressure')]
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
for variable in variables:
try:
var_data = read_many_files_individually(
filenames, variable[0])[variable[0]]
meta = get_metadata(var_data[0])
meta.standard_name = variable[2]
# Some of the variables have an illegal name attribute...
meta.misc.pop('name', None)
c = Coord(var_data, meta, axis=variable[1])
if variable[1] == 'z':
c.convert_units('m')
coords.append(c)
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
usr_var_data = read_many_files_individually(
filenames, usr_variable)[usr_variable]
meta = get_metadata(usr_var_data[0])
# Some of the variables have an illegal name attribute...
meta.misc.pop('name', None)
res = UngriddedData(usr_var_data, meta, coords)
return res
def _create_coord_list(self, filenames):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord
from cis.exceptions import InvalidVariableError
try:
variables = ["lon", "lat", "time"]
data = read_many_files_individually(filenames, variables)
except InvalidVariableError:
variables = ["longitude", "latitude", "time"]
data = read_many_files_individually(filenames, variables)
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
coords.append(Coord(data[variables[0]], get_metadata(data[variables[0]][0]), "X"))
coords.append(Coord(data[variables[1]], get_metadata(data[variables[1]][0]), "Y"))
coords.append(self._fix_time(Coord(data[variables[2]], get_metadata(data[variables[2]][0]), "T")))
return coords
def create_coords(self, filenames):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates
var_data = read_many_files_individually(filenames, ["longitude", "latitude", "time"])
lon = Coord(var_data["longitude"], get_metadata(var_data["longitude"][0]), axis="x")
lat = Coord(var_data["latitude"], get_metadata(var_data["latitude"][0]), axis="y")
time = Coord(var_data["time"], get_metadata(var_data["time"][0]), axis="t")
coords = CoordList([lat, lon, time])
return UngriddedCoordinates(coords)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
axis_lookup = {
"longitude": "x",
'latitude': 'y',
'altitude': 'z',
'time': 't',
'air_pressure': 'p'
}
coord_variables = [(v, axis_lookup[v]) for v in file_variables
if v in axis_lookup]
# Create a copy to contain all the variables to read
all_variables = list(coord_variables)
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames,
[v[0] for v in all_variables])
for name, axis in coord_variables:
try:
coords.append(
Coord(var_data[name],
get_metadata(var_data[name][0]),
axis=axis))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable],
get_metadata(var_data[usr_variable][0]),
coords)
return res
def create_data_object(self, filenames, variable):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedData
var_data = read_many_files_individually(filenames, ["longitude", "latitude", "time", variable])
lon = Coord(var_data["longitude"], get_metadata(var_data["longitude"][0]), axis="x")
lat = Coord(var_data["latitude"], get_metadata(var_data["latitude"][0]), axis="y")
time = Coord(var_data["time"], get_metadata(var_data["time"][0]), axis="t")
coords = CoordList([lat, lon, time])
usr_var_data = var_data[variable]
return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords)
def _create_coord_list(self, filenames):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord
variables = ["lat", "lon", "time"]
logging.info("Listing coordinates: " + str(variables))
var_data = read_many_files_individually(filenames, variables)
coords = CoordList()
coords.append(Coord(var_data['lat'], get_metadata(var_data['lat'][0]), 'Y'))
coords.append(Coord(var_data['lon'], get_metadata(var_data['lon'][0]), 'X'))
time_coord = Coord(var_data['time'], get_metadata(var_data['time'][0]))
# TODO: Is this really julian?
time_coord.convert_julian_to_std_time()
coords.append(time_coord)
return coords
def _load_data(self, filenames, variable):
"""
Open the file and find the correct variables to load in
:param filenames: the filenames to load
:param variable: an extra variable to load
:return: a list of load data and the variable selector used to load name it
"""
variable_selector = self._load_data_definition(filenames)
variables_list = [variable_selector.time_variable_name]
add_to_list_if_not_none(variable_selector.latitude_variable_name, variables_list)
add_to_list_if_not_none(variable_selector.longitude_variable_name, variables_list)
add_to_list_if_not_none(variable_selector.altitude_variable_name, variables_list)
add_to_list_if_not_none(variable_selector.pressure_variable_name, variables_list)
logging.info("Listing coordinates: " + str(variables_list))
add_to_list_if_not_none(variable, variables_list)
data_variables = read_many_files_individually(filenames, variables_list)
return data_variables, variable_selector
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
axis_lookup = {"longitude": "x", 'latitude': 'y', 'altitude': 'z', 'time': 't', 'air_pressure': 'p'}
coord_variables = [(v, axis_lookup[v]) for v in file_variables if v in axis_lookup]
# Create a copy to contain all the variables to read
all_variables = list(coord_variables)
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis in coord_variables:
try:
coords.append(Coord(var_data[name], get_metadata(var_data[name][0]), axis=axis))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates
from cis.exceptions import InvalidVariableError
variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"),
("time", "t"),
("aerosol_backscatter_coefficient", "m-1 sr-1")]
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
for variable in variables:
try:
var_data = read_many_files_individually(
filenames, variable[0])[variable[0]]
coords.append(
Coord(var_data,
get_metadata(var_data[0]),
axis=variable[1]))
except InvalidVariableError:
pass
return UngriddedCoordinates(coords)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates
from cis.exceptions import InvalidVariableError
variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"),
("time", "t"), ("relative_humidity", "RH"),
("surface_air_pressure", "Pa"), ("air_temprature", "K"),
("wind_speed"), ("Wind Diretion"), ("rainfall_rate")]
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
for variable in variables:
try:
var_data = read_many_files_individually(
filenames, variable[0])[variable[0]]
coords.append(
Coord(var_data,
get_metadata(var_data[0]),
axis=variable[1]))
except InvalidVariableError:
pass
return UngriddedCoordinates(coords)
def create_coords(self, filenames, variable=None):
"""
Override the default read-in to also read in CCN quality flag data and apply the appropriate mask. We have
to do this before creating the UngriddedData object so that the missing coords don't get fixed first
"""
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.utils import apply_mask_to_numpy_array, concatenate
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
data_variables, variable_selector = self._load_data(filenames, variable)
dim_coords = self._create_coordinates_list(data_variables, variable_selector)
if variable is None:
return UngriddedCoordinates(dim_coords)
else:
aux_coord_name = variable_selector.find_auxiliary_coordinate(variable)
if aux_coord_name is not None:
all_coords = self._add_aux_coordinate(dim_coords, filenames[0], aux_coord_name,
dim_coords.get_coord(standard_name='time').data.size)
else:
all_coords = dim_coords
var_data = data_variables[variable]
if variable and variable.startswith('CCN_COL'):
# Work out the associated variable name for this column
ccn_flag_var = "COL{}_FLAG".format(variable[-1])
# Read in the flags
flags = concatenate([get_data(v) for v in read_many_files_individually(filenames, ccn_flag_var)[
ccn_flag_var]])
# 0 and 1 are both OK
mask = flags > 1
# If a variable was supplied then coords must be an ungridded data object, apply the mask to it
var_data = apply_mask_to_numpy_array(concatenate([get_data(v) for v in var_data]), mask)
return UngriddedData(var_data, get_metadata(data_variables[variable][0]), all_coords)
def create_data_object(self, filenames, variable):
logging.debug("Creating data object for variable " + variable)
variables = [("ER2_IMU/Longitude", "x"), ("ER2_IMU/Latitude", "y"),
("ER2_IMU/gps_time", "t"), ("State/Pressure", "p"),
("DataProducts/Altitude", "z"), ("header/date", ""),
(variable, '')]
logging.info("Listing coordinates: " + str(variables))
var_data = read_many_files_individually(filenames,
[v[0] for v in variables])
date_times = []
for times, date in zip(var_data['ER2_IMU/gps_time'],
var_data['header/date']):
# Date is stored as an array (of length 92??) of floats with format: yyyymmdd
date_str = str(int(date[0]))
t_unit = Unit('hours since {}-{}-{} 00:00:00'.format(
date_str[0:4], date_str[4:6], date_str[6:8]))
date_times.append(
t_unit.convert(get_data(times), cis_standard_time_unit))
# time_data = utils.concatenate([get_data(i) for i in var_data['ER2_IMU/gps_time']])
# date_str = str(int(var_data['header/date'][0][0]))
# Flatten the data by taking the 0th column of the transpose
time_coord = DimCoord(utils.concatenate(date_times).T[0],
standard_name='time',
units=cis_standard_time_unit)
# TODO This won't work for multiple files since the altitude bins are different for each flight...
alt_data = utils.concatenate(
[get_data(i) for i in var_data["DataProducts/Altitude"]])
alt_coord = DimCoord(alt_data[0], standard_name='altitude', units='m')
pres_data = utils.concatenate(
[get_data(i) for i in var_data["State/Pressure"]])
pres_coord = AuxCoord(pres_data,
standard_name='air_pressure',
units='atm')
# Fix the air-pressure units
pres_coord.convert_units('hPa')
lat_data = utils.concatenate(
[get_data(i) for i in var_data['ER2_IMU/Latitude']])
lat_coord = AuxCoord(lat_data.T[0], standard_name='latitude')
lon_data = utils.concatenate(
[get_data(i) for i in var_data['ER2_IMU/Longitude']])
lon_coord = AuxCoord(lon_data.T[0], standard_name='longitude')
data = utils.concatenate([get_data(i) for i in var_data[variable]])
metadata = get_metadata(var_data[variable][0])
cube = Cube(np.ma.masked_invalid(data),
long_name=metadata.misc['Description'],
units=self.clean_units(metadata.units),
dim_coords_and_dims=[(alt_coord, 1), (time_coord, 0)],
aux_coords_and_dims=[(lat_coord, (0, )),
(lon_coord, (0, )),
(pres_coord, (0, 1))])
gd = GriddedData.make_from_cube(cube)
return gd
