def setUp(self):
x_points = np.arange(-10, 11, 5)
y_points = np.arange(-5, 6, 5)
y, x = np.meshgrid(y_points, x_points)
x = Coord(
x, Metadata(name='lat', standard_name='latitude', units='degrees'))
y = Coord(
y, Metadata(name='lon', standard_name='longitude',
units='degrees'))
data = np.reshape(np.arange(15) + 1.0, (5, 3))
self.coords = CoordList([x, y])
ug1 = UngriddedData(
data,
Metadata(standard_name='rainfall_flux',
long_name="TOTAL RAINFALL RATE: LS+CONV KG/M2/S",
units="kg m-2 s-1",
missing_value=-999), self.coords)
ug2 = UngriddedData(
data * 0.1,
Metadata(standard_name='snowfall_flux',
long_name="TOTAL SNOWFALL RATE: LS+CONV KG/M2/S",
units="kg m-2 s-1",
missing_value=-999), self.coords)
self.ungridded_data_list = UngriddedDataList([ug1, ug2])
def create_data_object(self, filenames, variable):
from cis.exceptions import InvalidVariableError, CISError
import numpy as np
try:
data_obj = load_multiple_hysplit(filenames, [variable])
except ValueError:
raise InvalidVariableError(variable + " does not exist in " +
str(filenames))
except EOFError as e:
raise CISError(e)
except IOError as e:
raise CISError(e) # TODO
coords = self._create_coord_list(filenames, data_obj)
# WRITE STANDARD NAME GUESSER HERE
if variable == "PRESSURE":
variable = "air_pressure"
elif variable == "RELHUMID":
variable = "relative_humidity"
objM = Metadata(name=variable,
standard_name=variable,
long_name=variable,
shape=(len(data_obj[variable]), ),
missing_value=-99999.0)
#objM.standard_name = None
#print((len(data_obj[variable]),))
return UngriddedData(data_obj[variable], objM, coords)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
variables = [("longitude", "x"), ("latitude", "y")]
# if usr_variable is not None:
# variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
var_data = read_many_files_individually(filenames,
[v[0] for v in variables])
for var, (name, axis) in zip(var_data.values(), variables):
try:
coords.append(Coord(var, get_metadata(var[0]), axis=axis))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
usr_var_data = read_many_files_individually(
filenames, usr_variable)[usr_variable]
res = UngriddedData(usr_var_data, get_metadata(usr_var_data[0]),
coords)
return res
def create_data_object(self, filenames, variable):
data_dict = {} #initialise data dictionary
inData = netCDF4.Dataset(filenames[0]) #open netCDF file
data_dict['longitude'] = np.array(
inData.variables['lon']) #extract longitudes
data_dict['latitude'] = np.array(
inData.variables['lat']) #extract latitudes
origTimes = np.array(inData.variables['time_counter']) #extract times
#Convert time to days since
niceDateTime = cf_units.num2date(origTimes,
'seconds since 1999-01-01 00:00:00',
'gregorian')
data_dict['time'] = cf_units.date2num(
niceDateTime, 'days since 1600-01-01 00:00:00', 'gregorian')
data_dict[variable] = np.array(
inData.variables[variable]) #extract requested variable
inData.close() #close netCDF file
coords = self._create_coord_list(filenames, data_dict)
return UngriddedData(
data_dict[variable],
Metadata(name=variable,
long_name=variable,
shape=(len(data_dict), ),
missing_value=-999.0,
units="1"), coords)
def test_GIVEN_multiple_ungridded_data_with_missing_data_WHEN_call_as_data_frame_THEN_returns_valid_data_frame(
self):
d = np.reshape(np.arange(15) + 10.0, (5, 3))
data = np.ma.masked_array(d, np.zeros(d.shape, dtype=bool))
data.mask[1, 2] = True
ug3 = UngriddedData(
data,
Metadata(name='hail',
long_name="TOTAL HAIL RATE: LS+CONV KG/M2/S",
units="kg m-2 s-1",
missing_value=-999), self.coords)
self.ungridded_data_list.append(ug3)
df = self.ungridded_data_list.as_data_frame()
assert_that(df['rainfall_flux'][5] == 6)
assert_almost_equal(df['snowfall_flux'][5], 0.6)
assert_that(df['latitude'][13] == 10)
assert_that(df['longitude'][0] == -5)
assert_almost_equal(df['TOTAL HAIL RATE: LS+CONV KG/M2/S'][1], 11.0)
assert_that(
np.isnan(
df['TOTAL HAIL RATE: LS+CONV KG/M2/S'][np.ravel_multi_index(
[1, 2], (5, 3))]))
self.ungridded_data_list.pop()
def create_coords(self, filenames, variable=None):
"""
Reads the coordinates and data if required from the files
:param filenames: List of filenames to read coordinates from
:param variable: load a variable for the data
:return: Coordinates
"""
from cis.data_io.netcdf import read_many_files_individually
from cis.data_io.Coord import Coord, CoordList
from cis.exceptions import InvalidVariableError
variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"), ("time", "t"), ("air_pressure", "p")]
dim_coords = CoordList()
for v in variables:
try:
var_data = read_many_files_individually(filenames, v[0])[v[0]]
dim_coords.append(Coord(var_data, get_metadata(var_data[0]), axis=v[1]))
except InvalidVariableError:
pass
if variable is None:
return UngriddedCoordinates(dim_coords)
else:
all_coords = self._add_aux_coordinate(dim_coords, filenames[0], 'DP_MID',
dim_coords.get_coord(standard_name='time').data.size)
usr_var_data = read_many_files_individually(filenames, variable)[variable]
return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), all_coords)
def create_coords(self, filenames, variable=None):
"""
Reads the coordinates and data if required from the files
:param filenames: List of filenames to read coordinates from
:param variable: load a variable for the data
:return: Coordinates
"""
data_variables, variable_selector = self._load_data(
filenames, variable)
dim_coords = self._create_coordinates_list(data_variables,
variable_selector)
if variable is None:
return UngriddedCoordinates(dim_coords)
else:
aux_coord_name = variable_selector.find_auxiliary_coordinate(
variable)
if aux_coord_name is not None:
all_coords = self._add_aux_coordinate(
dim_coords, filenames[0], aux_coord_name,
dim_coords.get_coord(standard_name='time').data.size)
else:
all_coords = dim_coords
return UngriddedData(data_variables[variable],
get_metadata(data_variables[variable][0]),
all_coords)
def stack_data_list(data_list, var_name=None, units=None):
"""
Stacks a list of Ungridded data objects with the same data variable, but different coordinates into a single
UngriddedData object, with accompanying lat, lon and time data.
It assumes the coordinates have the same *sets* of coordinates, but that the coordinates themselves are different.
:param data_list: list of UngriddedData objects to be merged (stacked vertically)
:param string var_name: Name of final data variable
:param string units: Units of final data variable
:return: A merged UngriddedData object
"""
import numpy as np
from cis.data_io.ungridded_data import UngriddedData, Metadata
from cis.data_io.Coord import Coord
coords = []
all_data = np.hstack((d.data_flattened for d in data_list))
# Create mask (not all datasets might be masked)
mask = np.hstack((d.data.mask.ravel() if isinstance(
d.data, np.ma.MaskedArray) else np.zeros(d.shape, np.bool)
for d in data_list))
all_data = np.ma.MaskedArray(all_data, mask)
for c in data_list[0].coords():
coord_data = np.hstack((d.coord(c).data_flattened for d in data_list))
coords.append(Coord(coord_data, c.metadata, axis=c.axis))
return UngriddedData(
all_data,
Metadata(name=var_name or data_list[0].name(),
units=units or data_list[0].units), coords)
def create_data_object(self, filenames, variable):
from cis.data_io.netcdf import get_metadata, read_many_files_individually
coords = self._create_coord_list(filenames)
var = read_many_files_individually(filenames, [variable])
metadata = get_metadata(var[variable][0])
return UngriddedData(var[variable], metadata, coords)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
def get_axis_std_name(var):
axis=None
lvar = var.lower()
if lvar == 'longitude':
axis = 'x', 'longitude'
if lvar == 'latitude':
axis = 'y', 'latitude'
if lvar == 'G_ALT' or lvar == 'altitude' or lvar == 'pressure_altitude':
axis = 'z', 'altitude'
if lvar == 'time':
axis = 't', 'time'
if lvar == 'p' or lvar == 'pressure' or lvar == 'static_pressure':
axis = 'p', 'air_pressure'
return axis
all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None]
# Get rid of any duplicates
coord_variables = []
for v in all_coord_variables:
if v is None or v[1][1] not in [x[1][1] for x in coord_variables]:
coord_variables.append(v)
all_variables = coord_variables.copy()
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis_std_name in coord_variables:
try:
meta = get_metadata(var_data[name][0])
if meta.standard_name is None:
meta.standard_name = axis_std_name[1]
coords.append(Coord(var_data[name], meta, axis=axis_std_name[0]))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
def create_data_object(self, filenames, variable):
from cis.data_io.ungridded_data import UngriddedData
usr_var_data = read_many_files_individually(filenames,
variable)[variable]
coords = self.create_coords(filename)
return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]),
coords)
def collocate(self, points, data, constraint, kernel):
"""
This collocator takes a list of HyperPoints and a data object (currently either Ungridded
data or a Cube) and returns one new LazyData object with the values as determined by the
constraint and kernel objects. The metadata for the output LazyData object is copied from
the input data object.
:param UngriddedData or UngriddedCoordinates points: Objects defining the sample points
:param GriddedData or GriddedDataList data: Data to resample
:param constraint: An instance of a Constraint subclass which takes a data object and
returns a subset of that data based on it's internal parameters
:param kernel: An instance of a Kernel subclass which takes a number of points and returns
a single value
:return: A single LazyData object
"""
from cis.collocation.gridded_interpolation import GriddedUngriddedInterpolator
log_memory_profile("GriddedUngriddedCollocator Initial")
if isinstance(data, list):
# Indexing and constraints (for SepConstraintKdTree) will only take place on the first iteration,
# so we really can just call this method recursively if we've got a list of data.
output = UngriddedDataList()
for var in data:
output.extend(self.collocate(points, var, constraint, kernel))
return output
if constraint is not None and not isinstance(constraint, DummyConstraint):
raise ValueError("A constraint cannot be specified for the GriddedUngriddedCollocator")
data_points = data
# First fix the sample points so that they all fall within the same 360 degree longitude range
_fix_longitude_range(points.coords(), points)
# Then fix the data points so that they fall onto the same 360 degree longitude range as the sample points
_fix_longitude_range(points.coords(), data_points)
log_memory_profile("GriddedUngriddedCollocator after data retrieval")
logging.info("--> Collocating...")
logging.info(" {} sample points".format(points.size))
if self.interpolator is None:
# Cache the interpolator
self.interpolator = GriddedUngriddedInterpolator(data, points, kernel, self.missing_data_for_missing_sample)
values = self.interpolator(data, fill_value=self.fill_value, extrapolate=self.extrapolate)
log_memory_profile("GriddedUngriddedCollocator after running kernel on sample points")
metadata = Metadata(self.var_name or data.var_name, long_name=self.var_long_name or data.long_name,
shape=values.shape, missing_value=self.fill_value, units=self.var_units or data.units)
set_standard_name_if_valid(metadata, data.standard_name)
return_data = UngriddedDataList([UngriddedData(values, metadata, points.coords())])
log_memory_profile("GriddedUngriddedCollocator final")
return return_data
def create_data_object(self, filenames, variable):
logging.info("gathering data for variable " + str(variable))
for filename in filenames:
data = []
logging.info("gatherings metadata for variable " + str(variable))
metadata = Metadata()
coords = self.create_coords(filenames)
return UngriddedData(data, metadata, coords)
def scene_to_cis(scn):
from cis.data_io.Coord import Coord
from cis.data_io.ungridded_data import Metadata, UngriddedData
data = scn['M05']
area = data.info['area']
units = data.info['units']
lats = Coord(
area.lats, Metadata(standard_name='latitude', units='degrees'), 'y')
lons = Coord(
area.lons, Metadata(standard_name='longitude', units='degrees'), 'x')
print(area.lons.info)
ug = UngriddedData(data, Metadata(name=data.info['name'], units=units), [lats, lons])
return ug
def make_mock_CALIOP_data(data, name='', units=''):
from cis.utils import expand_1d_to_2d_array
vertical_levels, swath_length = data.shape
lat = Coord(
expand_1d_to_2d_array(np.arange(swath_length), vertical_levels),
Metadata(standard_name='latitude'))
lon = Coord(
expand_1d_to_2d_array(np.arange(swath_length), vertical_levels),
Metadata(standard_name='longitude'))
alt = Coord(
expand_1d_to_2d_array(np.arange(vertical_levels), swath_length,
axis=1), Metadata(standard_name='altitude'))
return UngriddedData(data, Metadata(name, units=units), [lat, lon, alt])
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
axis_lookup = {
"longitude": "x",
'latitude': 'y',
'altitude': 'z',
'time': 't',
'air_pressure': 'p'
}
coord_variables = [(v, axis_lookup[v]) for v in file_variables
if v in axis_lookup]
# Create a copy to contain all the variables to read
all_variables = list(coord_variables)
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames,
[v[0] for v in all_variables])
for name, axis in coord_variables:
try:
coords.append(
Coord(var_data[name],
get_metadata(var_data[name][0]),
axis=axis))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable],
get_metadata(var_data[usr_variable][0]),
coords)
return res
def create_data_object(self, filenames, variable):
from itertools import product
logging.debug("Creating data object for variable " + variable)
# reading coordinates
# the variable here is needed to work out whether to apply interpolation to the lat/lon data or not
coords = self._create_coord_list(filenames, variable)
# reading of variables
sdata, vdata = hdf.read(filenames, variable)
# retrieve data + its metadata
var = sdata[variable]
metadata = hdf.read_metadata(var, "SD")
# Check the dimension of this variable
_, ndim, dim_len, _, _ = var[0].info()
if ndim == 2:
return UngriddedData(var, metadata, coords, _get_MODIS_SDS_data)
elif ndim < 2:
raise NotImplementedError("1D field in MODIS L2 data.")
else:
result = UngriddedDataList()
# Iterate over all but the last two dimensions
ranges = [range(n) for n in dim_len[:-2]]
for indices in product(*ranges):
for manager in var:
manager._start = list(indices) + [0, 0]
manager._count = [1
] * len(indices) + manager.info()[2][-2:]
result.append(
UngriddedData(var, metadata, coords.copy(),
_get_MODIS_SDS_data))
return result
def create_data_object(self, filenames, variable):
from pywork.CALIOP_utils import mask_data
logging.debug("Creating *QC'd* data object for variable " + variable)
# reading of variables
sdata, vdata = hdf.read(filenames, [variable, "Pressure", "Extinction_QC_Flag_532", "CAD_Score"])
# retrieve data + its metadata
var = sdata[variable]
metadata = hdf.read_metadata(var, "SD")
# reading coordinates
# See if the variable is one dimensional (check the length of shape, neglecting length 1 dimensions)
if len([l for l in metadata.shape if l > 1]) == 1:
coords = self._create_one_dimensional_coord_list(filenames, index_offset=1)
else:
coords = self._create_coord_list(filenames, index_offset=1)
if variable in MIXED_RESOLUTION_VARIABLES:
logging.warning("Using Level 2 resolution profile for mixed resolution variable {}. See CALIPSO "
"documentation for more details".format(variable))
callback = self._get_mixed_resolution_calipso_data
else:
callback = self._get_calipso_data
var_data = hdf.read_data(sdata[variable], callback)
extinction_qc = hdf.read_data(sdata["Extinction_QC_Flag_532"], self._get_mixed_resolution_calipso_data)
cad_score = hdf.read_data(sdata["CAD_Score"], self._get_mixed_resolution_calipso_data)
qcd_var_data, = mask_data(var_data, cad_score, extinction_qc)
# reading coordinates
if variable.startswith('Column'):
coords = self._create_one_dimensional_coord_list(filenames, index_offset=1)
else:
coords = self._create_coord_list(filenames, index_offset=1)
pres_data = hdf.read_data(sdata['Pressure'], self._get_calipso_data)
pres_metadata = hdf.read_metadata(sdata['Pressure'], "SD")
# Fix badly formatted units which aren't CF compliant and will break if they are aggregated
if str(pres_metadata.units) == "hPA":
pres_metadata.units = "hPa"
qcd_pres_data = mask_data(pres_data, cad_score, extinction_qc)
pres_coord = Coord(qcd_pres_data, pres_metadata, 'P')
coords.append(pres_coord)
return UngriddedData(qcd_var_data, metadata, coords)
def create_data_object(self, filenames, variable):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedData
var_data = read_many_files_individually(filenames, ["longitude", "latitude", "time", variable])
lon = Coord(var_data["longitude"], get_metadata(var_data["longitude"][0]), axis="x")
lat = Coord(var_data["latitude"], get_metadata(var_data["latitude"][0]), axis="y")
time = Coord(var_data["time"], get_metadata(var_data["time"][0]), axis="t")
coords = CoordList([lat, lon, time])
usr_var_data = var_data[variable]
return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords)
def create_data_object(self, filenames, variable):
logging.debug("Creating data object for variable " + variable)
# reading coordinates
# the variable here is needed to work out whether to apply interpolation to the lat/lon data or not
coords = self._create_coord_list(filenames, variable)
# reading of variables
sdata, vdata = hdf.read(filenames, variable)
# retrieve data + its metadata
var = sdata[variable]
metadata = hdf.read_metadata(var, "SD")
return UngriddedData(var, metadata, coords, _get_MODIS_SDS_data)
def test_GIVEN_UngriddedDataList_WHEN_constrain_THEN_correctly_subsetted_UngriddedDataList_returned(
self):
ug_data = cis.test.util.mock.make_regular_2d_ungridded_data()
ug_data2 = UngriddedData(
ug_data.data + 1,
Metadata(name='snow',
standard_name='snowfall_flux',
long_name="TOTAL SNOWFALL RATE: LS+CONV KG/M2/S",
units="kg m-2 s-1",
missing_value=-999), ug_data.coords())
datalist = UngriddedDataList([ug_data, ug_data2])
subset = datalist.subset(longitude=[0.0, 5.0], latitude=[-5.0, 10.0])
assert isinstance(subset, UngriddedDataList)
assert subset[0].data.tolist() == [5, 6, 8, 9, 11, 12, 14, 15]
assert subset[1].data.tolist() == [6, 7, 9, 10, 12, 13, 15, 16]
def test_GIVEN_missing_coord_values_WHEN_data_flattened_THEN_missing_values_removed(
self):
x_points = np.arange(-10, 11, 5)
y_points = np.arange(-5, 6, 5)
y, x = np.meshgrid(y_points, x_points)
y = np.ma.masked_array(y, np.zeros(y.shape, dtype=bool))
y.mask[1, 2] = True
x = Coord(x, Metadata(standard_name='latitude', units='degrees'))
y = Coord(y, Metadata(standard_name='longitude', units='degrees'))
coords = CoordList([x, y])
data = np.reshape(np.arange(15) + 1.0, (5, 3))
ug = UngriddedData(None, Metadata(), coords, lambda x: data)
data = ug.data_flattened
assert_that(len(data), is_(14))
def create_data_object(self, filenames, variable):
logging.debug("Creating data object for variable " + variable)
# reading coordinates
# the variable here is needed to work out whether to apply interpolation to the lat/lon data or not
coords = self._create_coord_list(filenames[0])
usr_var_data = read_many_files_individually(filenames,
variable)[variable]
qc_flag = read_many_files_individually(filenames, "qc_flag")["qc_flag"]
import numpy as np
mask = (qc_flag[0][:, :, 0] != 1)
#retdata = np.ma.array(usr_var_data[0][:, :, 0],mask = mask)
from cis.utils import apply_mask_to_numpy_array, concatenate
retdata = apply_mask_to_numpy_array(usr_var_data[0][:, :, 0], mask)
return UngriddedData(retdata, get_metadata(usr_var_data[0]), coords)
def create_data_object(self, filenames, variable):
from cis.data_io.aeronet import load_multiple_aeronet
data_obj = load_multiple_aeronet(filenames, [variable])
coords = self._create_coord_list(filenames, data_obj)
# The name is text before any brackets, the units is what's after it (minus the closing bracket)
name_units = variable.split('(')
name = name_units[0]
# For Aeronet we can assume that if there are no units then it is unitless (AOT, Angstrom exponent, etc)
units = name_units[1][:-1] if len(name_units) > 1 else '1'
return UngriddedData(
data_obj[variable],
Metadata(name=name,
long_name=variable,
shape=(len(data_obj), ),
units=units,
missing_value=-999.0), coords)
def create_coords(self, filenames, variable=None):
from cis.data_io.ungridded_data import Metadata
from numpy import genfromtxt, NaN
from cis.exceptions import InvalidVariableError
from cis.time_util import convert_datetime_to_std_time
import dateutil.parser as du
array_list = []
for filename in filenames:
try:
array_list.append(genfromtxt(filename, dtype="f8,f8,f8,O,f8",
names=['latitude', 'longitude', 'altitude', 'time', 'value'],
delimiter=',', missing_values='', usemask=True, invalid_raise=True,
converters={"time": du.parse}))
except:
raise IOError('Unable to read file ' + filename)
data_array = utils.concatenate(array_list)
n_elements = len(data_array['latitude'])
coords = CoordList()
coords.append(Coord(data_array["latitude"],
Metadata(standard_name="latitude", shape=(n_elements,), units="degrees_north")))
coords.append(Coord(data_array["longitude"],
Metadata(standard_name="longitude", shape=(n_elements,), units="degrees_east")))
coords.append(
Coord(data_array["altitude"], Metadata(standard_name="altitude", shape=(n_elements,), units="meters")))
time_arr = convert_datetime_to_std_time(data_array["time"])
time = Coord(time_arr,
Metadata(standard_name="time", shape=(n_elements,), units="days since 1600-01-01 00:00:00"))
coords.append(time)
if variable:
try:
data = UngriddedData(data_array['value'], Metadata(name="value", shape=(n_elements,), units="unknown",
missing_value=NaN), coords)
except:
InvalidVariableError("Value column does not exist in file " + filenames)
return data
else:
return UngriddedCoordinates(coords)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.data_io.Coord import Coord, CoordList
from cis.exceptions import InvalidVariableError
variables = [("lon", "x", 'longitude'), ("lat", "y", 'latitude'),
("alt", "z", 'altitude'), ("time", "t", 'time'),
("p", "p", 'air_pressure')]
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
for variable in variables:
try:
var_data = read_many_files_individually(
filenames, variable[0])[variable[0]]
meta = get_metadata(var_data[0])
meta.standard_name = variable[2]
# Some of the variables have an illegal name attribute...
meta.misc.pop('name', None)
c = Coord(var_data, meta, axis=variable[1])
if variable[1] == 'z':
c.convert_units('m')
coords.append(c)
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
usr_var_data = read_many_files_individually(
filenames, usr_variable)[usr_variable]
meta = get_metadata(usr_var_data[0])
# Some of the variables have an illegal name attribute...
meta.misc.pop('name', None)
res = UngriddedData(usr_var_data, meta, coords)
return res
def create_data_object(self, filenames, variable):
logging.debug("Creating data object for variable " + variable)
# reading of variables
sdata, vdata = hdf.read(filenames, variable)
# reading (un-expanded) coordinates, since the data is 1-dimensional
coords = self._create_one_dimensional_coord_list(filenames)
# retrieve data + its metadata
if variable in vdata:
var = hdf.read_data(vdata[variable], self._get_cloudsat_vds_data)
metadata = hdf.read_metadata(vdata[variable], "VD")
elif variable in sdata:
var = hdf.read_data(sdata[variable], self._get_cloudsat_sds_data)
metadata = hdf.read_metadata(sdata[variable], "SD")
else:
raise ValueError("variable not found")
return UngriddedData(var, metadata, coords)
def create_data_object(self, filenames, variable):
logging.debug("Creating data object for variable " + variable)
# reading coordinates
# the variable here is needed to work out whether to apply interpolation to the lat/lon data or not
coords = self._create_coord_list(filenames, variable)
# reading of variables
sdata, vdata = hdf.read(filenames, variable)
# retrieve data + its metadata
var = sdata[variable]
metadata = hdf.read_metadata(var, "SD")
# cut off the edges of the data...
# TODO CHECK THIS IS ACTUALLY VALID BEFORE PUBLISHING ANYTHING WITH THIS
d = hdf.read_data(var, _get_MODIS_SDS_data)
if self.apply_interpolation:
d = d[:, 2:-2]
return UngriddedData(d, metadata, coords, _get_MODIS_SDS_data)
def create_data_object(self, filenames, variable):
from pywork.icartt_utils import _readict
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedData
# FIXME
var_data = read_many_files_individually(
filenames, [variable, "longitude", "latitude", "time"])
lon = Coord(var_data["longitude"],
get_metadata(var_data["longitude"][0]),
axis="x")
lat = Coord(var_data["latitude"],
get_metadata(var_data["latitude"][0]),
axis="y")
time = Coord(var_data["time"],
get_metadata(var_data["time"][0]),
axis="t")
coords = CoordList([lon, lat, time])
return UngriddedData(var_data[variable],
get_metadata(var_data[variable][0]), coords)
def create_data_object(self, filenames, variable):
logging.debug("Creating data object for variable " + variable)
# reading coordinates
if variable.startswith('Column'):
coords = self._create_one_dimensional_coord_list(filenames, index_offset=1)
else:
coords = self._create_coord_list(filenames, index_offset=1)
# reading of variables
sdata, vdata = hdf.read(filenames, variable)
# retrieve data + its metadata
var = sdata[variable]
metadata = hdf.read_metadata(var, "SD")
if variable in MIXED_RESOLUTION_VARIABLES:
logging.warning("Using Level 2 resolution profile for mixed resolution variable {}. See CALIPSO "
"documentation for more details".format(variable))
callback = self._get_mixed_resolution_calipso_data
else:
callback = self._get_calipso_data
return UngriddedData(var, metadata, coords, callback)
