def setUp(self):
x_points = np.arange(-10, 11, 5)
y_points = np.arange(-5, 6, 5)
y, x = np.meshgrid(y_points, x_points)
x = Coord(
x, Metadata(name='lat', standard_name='latitude', units='degrees'))
y = Coord(
y, Metadata(name='lon', standard_name='longitude',
units='degrees'))
data = np.reshape(np.arange(15) + 1.0, (5, 3))
self.coords = CoordList([x, y])
ug1 = UngriddedData(
data,
Metadata(standard_name='rainfall_flux',
long_name="TOTAL RAINFALL RATE: LS+CONV KG/M2/S",
units="kg m-2 s-1",
missing_value=-999), self.coords)
ug2 = UngriddedData(
data * 0.1,
Metadata(standard_name='snowfall_flux',
long_name="TOTAL SNOWFALL RATE: LS+CONV KG/M2/S",
units="kg m-2 s-1",
missing_value=-999), self.coords)
self.ungridded_data_list = UngriddedDataList([ug1, ug2])
def can_not_convert_time_without_since_in_units_with_no_units():
times = numpy.array([0, 1])
units = ""
time_stamp_info = "1601-01-01"
coord = Coord(times, Metadata(units=units))
coord.convert_to_std_time(time_stamp_info)
def _create_coord_list(self, filename):
import numpy as np
coords = CoordList()
time_data = read(filename, 'time')['time']
len_x = time_data.shape[0]
try:
alt_data = read(filename, 'altitude')['altitude']
except InvalidVariableError:
alt_data = read(filename, 'range')['range']
len_y = alt_data.shape[0]
time_arr = utils.expand_1d_to_2d_array(time_data[:], len_y, axis=1)
t_coord = Coord(time_arr, get_metadata(time_data), axis='x')
t_coord.convert_to_std_time()
coords.append(t_coord)
alt_arr = utils.expand_1d_to_2d_array(alt_data[:], len_x, axis=0)
coords.append(Coord(alt_arr, get_metadata(alt_data), axis='y'))
lat_data = read(filename, 'latitude')['latitude']
lat_arr = np.ones(alt_arr.shape) * lat_data[:]
coords.append(Coord(lat_arr, get_metadata(lat_data)))
lon_data = read(filename, 'longitude')['longitude']
lon_arr = np.ones(alt_arr.shape) * lon_data[:]
coords.append(Coord(lon_arr, get_metadata(lon_data)))
return coords
def _create_coord_list(self, filenames):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord
from cis.exceptions import InvalidVariableError
try:
variables = ["lon", "lat", "time"]
data = read_many_files_individually(filenames, variables)
except InvalidVariableError:
variables = ["longitude", "latitude", "time"]
data = read_many_files_individually(filenames, variables)
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
coords.append(
Coord(data[variables[0]], get_metadata(data[variables[0]][0]),
"X"))
coords.append(
Coord(data[variables[1]], get_metadata(data[variables[1]][0]),
"Y"))
coords.append(
self._fix_time(
Coord(data[variables[2]], get_metadata(data[variables[2]][0]),
"T")))
return coords
def create_data_object(self, filenames, variable):
from pywork.CALIOP_utils import mask_data
# Read all the vars without pressure
var_data = super().create_data_object(filenames, variable)
extinction_qc = super().create_data_object(filenames, "Extinction_QC_Flag_532")
cad_score = super().create_data_object(filenames, "CAD_Score")
qcd_var_data, = mask_data([var_data], cad_score, extinction_qc)
if not variable.startswith('Column'):
# Read pressure separately, as it's own variable
pressure = super().create_data_object(filenames, "Pressure")
qcd_pres, = mask_data([pressure], cad_score, extinction_qc)
qcd_pres_coord = Coord(qcd_pres.data, qcd_pres.metadata, 'P')
# Fix badly formatted units which aren't CF compliant and will break if they are aggregated
if str(qcd_pres_coord.units) == "hPA":
qcd_pres_coord.units = "hPa"
qcd_var_data._coords.append(qcd_pres_coord)
qcd_var_data._post_process()
return qcd_var_data
def can_not_convert_time_without_since_in_units_with_no_units():
times = numpy.array([0, 1])
units = ""
time_stamp_info = "1601-01-01"
coord = Coord(times, Metadata(units=units))
coord.convert_to_std_time(time_stamp_info)
def _create_time_coord(self, timestamp, time_variable_name, data_variables, coord_axis='T', standard_name='time'):
"""
Create a time coordinate, taking into account the fact that each file may have a different timestamp.
:param timestamp: Timestamp or list of timestamps for
:param time_variable_name: Name of the time variable
:param data_variables: Dictionary containing one or multiple netCDF data variables for each variable name
:param coord_axis: Axis, default 'T'
:param standard_name: Coord standard name, default 'time'
:return: Coordinate
"""
from cis.data_io.Coord import Coord
from six.moves import zip_longest
timestamps = listify(timestamp)
time_variables = data_variables[time_variable_name]
time_coords = []
# Create a coordinate for each separate file to account for differing timestamps
for file_time_var, timestamp in zip_longest(time_variables, timestamps):
metadata = get_metadata(file_time_var)
metadata.standard_name = standard_name
coord = Coord(file_time_var, metadata, coord_axis)
coord.convert_to_std_time(timestamp)
time_coords.append(coord)
return Coord.from_many_coordinates(time_coords)
def _create_coord_list(self, filenames, variable=None):
import datetime as dt
variables = ["Latitude", "Longitude", "Scan_Start_Time"]
logging.info("Listing coordinates: " + str(variables))
sdata, vdata = hdf.read(filenames, variables)
apply_interpolation = False
if variable is not None:
scale = self.__get_data_scale(filenames[0], variable)
apply_interpolation = True if scale is "1km" else False
lat = sdata["Latitude"]
sd_lat = hdf.read_data(lat, "SD")
lat_data = self.__field_interpolate(sd_lat) if apply_interpolation else sd_lat
lat_metadata = hdf.read_metadata(lat, "SD")
lat_coord = Coord(lat_data, lat_metadata, "Y")
lon = sdata["Longitude"]
lon_data = (
self.__field_interpolate(hdf.read_data(lon, "SD")) if apply_interpolation else hdf.read_data(lon, "SD")
)
lon_metadata = hdf.read_metadata(lon, "SD")
lon_coord = Coord(lon_data, lon_metadata, "X")
time = sdata["Scan_Start_Time"]
time_metadata = hdf.read_metadata(time, "SD")
# Ensure the standard name is set
time_metadata.standard_name = "time"
time_coord = Coord(time, time_metadata, "T")
time_coord.convert_TAI_time_to_std_time(dt.datetime(1993, 1, 1, 0, 0, 0))
return CoordList([lat_coord, lon_coord, time_coord])
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
def get_axis_std_name(var):
axis=None
lvar = var.lower()
if lvar.startswith('lon'):
axis = 'x', 'longitude'
if lvar.startswith('lat'):
axis = 'y', 'latitude'
if lvar == 'G_ALT' or lvar == 'altitude' or lvar == 'pressure_altitude':
axis = 'z', 'altitude'
if lvar == 'time':
axis = 't', 'time'
if lvar == 'p' or lvar == 'pressure' or lvar == 'static_pressure':
axis = 'p', 'air_pressure'
return axis
all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None]
# Get rid of any duplicates
coord_variables = []
for v in all_coord_variables:
if v is None or v[1][1] not in [x[1][1] for x in coord_variables]:
coord_variables.append(v)
all_variables = coord_variables.copy()
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis_std_name in coord_variables:
try:
meta = get_metadata(var_data[name][0])
if meta.standard_name is None:
meta.standard_name = axis_std_name[1]
coord = Coord(var_data[name], meta, axis=axis_std_name[0])
if meta.standard_name == 'time':
# Converting units to CIS std time
coord.convert_to_std_time()
coords.append(coord)
except InvalidVariableError:
pass
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
def can_convert_time_with_since_in_units():
times = numpy.array([0, 1])
units = "Days since 1601-01-01"
coord = Coord(times, Metadata(units=units))
coord.convert_to_std_time()
assert_that(coord.data_flattened[0], is_(366.0), "time point")
def can_convert_time_with_since_in_units():
times = numpy.array([0, 1])
units = "days since 1601-01-01"
coord = Coord(times, Metadata(units=units))
coord.convert_to_std_time()
assert_that(coord.data_flattened[0], is_(366.0), "time point")
def create_dummy_coordinates_list():
coord1 = Coord(numpy.array([5, 4]),
Metadata(standard_name='grid_latitude'),
axis='Y')
coord2 = Coord(numpy.array([5, 4]),
Metadata(standard_name='grid_longitude'),
axis='X')
return CoordList([coord1, coord2])
def can_convert_time_without_since_in_units():
times = numpy.array([0, 1])
units = "Days from the file reference point 1601-01-01"
time_stamp_info = "1601-01-01"
coord = Coord(times, Metadata(units=units))
coord.convert_to_std_time(time_stamp_info)
assert_that(coord.data_flattened[0], is_(366.0), "time point")
def can_convert_time_without_since_in_units():
times = numpy.array([0, 1])
units = "Days from the file reference point 1601-01-01"
time_stamp_info = "1601-01-01"
coord = Coord(times, Metadata(units=units))
coord.convert_to_std_time(time_stamp_info)
assert_that(coord.data_flattened[0], is_(366.0), "time point")
def test_order_of_coords_doesnt_matter(self):
from iris.cube import Cube
from iris.coords import DimCoord
import numpy as np
from cis.data_io.gridded_data import make_from_cube
from cis.data_io.ungridded_data import UngriddedCoordinates, Metadata
from cis.data_io.Coord import Coord
cube_lat = DimCoord(np.linspace(-90, 90, 18),
standard_name='latitude',
units='degrees')
cube_lon = DimCoord(np.linspace(0, 359, 36),
standard_name='longitude',
units='degrees',
circular=True)
cube_alt = DimCoord(np.linspace(0, 10000, 10),
standard_name='altitude',
units='meters')
times = np.linspace(0, 30, 12)
cube_time = DimCoord(times,
standard_name='time',
units='days since 1970-01-01 00:00:00')
data = np.arange(12 * 18 * 36 * 10).reshape(12, 18, 36, 10)
source = make_from_cube(
Cube(data,
dim_coords_and_dims=[(cube_time, 0), (cube_lat, 1),
(cube_lon, 2), (cube_alt, 3)]))
n = 10
sample_lats = np.linspace(-9.1, 9.9, n)
sample_longs = np.linspace(-9.1, 9.9, n)
sample_alts = np.linspace(99, 599, n)
sample_times = np.linspace(0, 30, n)
sample = UngriddedCoordinates([
Coord(sample_lats, Metadata('latitude')),
Coord(sample_longs, Metadata('longitude')),
Coord(sample_alts, Metadata('altitude')),
Coord(sample_times,
Metadata('time', units='days since 1970-01-01 00:00:00'))
])
col = GriddedUngriddedCollocator()
output = col.collocate(sample, source, None, 'nn')[0]
source.transpose()
col = GriddedUngriddedCollocator()
assert_equal(
col.collocate(sample, source, None, 'nn')[0].data, output.data)
source.transpose((2, 1, 0, 3))
col = GriddedUngriddedCollocator()
assert_equal(
col.collocate(sample, source, None, 'nn')[0].data, output.data)
def setUp(self):
x_points = np.arange(-10, 11, 5)
y_points = np.arange(-5, 6, 5)
y, x = np.meshgrid(y_points, x_points)
self.x = Coord(x, Metadata(standard_name='latitude', units='degrees'))
self.y = Coord(y, Metadata(standard_name='longitude', units='degrees'))
coords = CoordList([self.x, self.y])
self.ug = UngriddedCoordinates(coords)
def can_find_many_coords_from_a_list_of_coordinates():
list = create_dummy_coordinates_list()
list.append(Coord(numpy.array([5, 4]), Metadata(name='testZ'), axis='Z'))
list.append(
Coord(numpy.array([5, 4]),
Metadata(name='testZ', standard_name='air_pressure')))
assert (len(list) == 4)
coords = list.get_coords(var_name='testZ')
assert (len(coords) == 2)
assert (coords[0].axis == 'Z')
assert (coords[1].axis == '')
assert (coords[1].name() == 'air_pressure')
def _create_one_dimensional_coord_list(self, filenames, index_offset=1):
"""
Create a set of coordinates appropriate for a ond-dimensional (column integrated) variable
:param filenames:
:param int index_offset: For 5km products this will choose the coordinates which represent the start (0),
middle (1) and end (2) of the 15 shots making up each column retrieval.
:return:
"""
from pyhdf.error import HDF4Error
from cis.data_io import hdf_sd
import datetime as dt
from cis.time_util import convert_sec_since_to_std_time, cis_standard_time_unit
variables = ['Latitude', 'Longitude', "Profile_Time"]
logging.info("Listing coordinates: " + str(variables))
# reading data from files
sdata = {}
for filename in filenames:
try:
sds_dict = hdf_sd.read(filename, variables)
except HDF4Error as e:
raise IOError(str(e))
for var in list(sds_dict.keys()):
utils.add_element_to_list_in_dict(sdata, var, sds_dict[var])
# latitude
lat_data = hdf.read_data(sdata['Latitude'], self._get_calipso_data)[:, index_offset]
lat_metadata = hdf.read_metadata(sdata['Latitude'], "SD")
lat_coord = Coord(lat_data, lat_metadata, 'Y')
# longitude
lon = sdata['Longitude']
lon_data = hdf.read_data(lon, self._get_calipso_data)[:, index_offset]
lon_metadata = hdf.read_metadata(lon, "SD")
lon_coord = Coord(lon_data, lon_metadata, 'X')
# profile time, x
time = sdata['Profile_Time']
time_data = hdf.read_data(time, self._get_calipso_data)[:, index_offset]
time_data = convert_sec_since_to_std_time(time_data, dt.datetime(1993, 1, 1, 0, 0, 0))
time_coord = Coord(time_data, Metadata(name='Profile_Time', standard_name='time', shape=time_data.shape,
units=cis_standard_time_unit), "T")
# create the object containing all coordinates
coords = CoordList()
coords.append(lat_coord)
coords.append(lon_coord)
coords.append(time_coord)
return coords
def create_coords(self, filenames):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates
var_data = read_many_files_individually(filenames, ["longitude", "latitude", "time"])
lon = Coord(var_data["longitude"], get_metadata(var_data["longitude"][0]), axis="x")
lat = Coord(var_data["latitude"], get_metadata(var_data["latitude"][0]), axis="y")
time = Coord(var_data["time"], get_metadata(var_data["time"][0]), axis="t")
coords = CoordList([lat, lon, time])
return UngriddedCoordinates(coords)
def scene_to_cis(scn):
from cis.data_io.Coord import Coord
from cis.data_io.ungridded_data import Metadata, UngriddedData
data = scn['M05']
area = data.info['area']
units = data.info['units']
lats = Coord(
area.lats, Metadata(standard_name='latitude', units='degrees'), 'y')
lons = Coord(
area.lons, Metadata(standard_name='longitude', units='degrees'), 'x')
print(area.lons.info)
ug = UngriddedData(data, Metadata(name=data.info['name'], units=units), [lats, lons])
return ug
def make_mock_CALIOP_data(data, name='', units=''):
from cis.utils import expand_1d_to_2d_array
vertical_levels, swath_length = data.shape
lat = Coord(
expand_1d_to_2d_array(np.arange(swath_length), vertical_levels),
Metadata(standard_name='latitude'))
lon = Coord(
expand_1d_to_2d_array(np.arange(swath_length), vertical_levels),
Metadata(standard_name='longitude'))
alt = Coord(
expand_1d_to_2d_array(np.arange(vertical_levels), swath_length,
axis=1), Metadata(standard_name='altitude'))
return UngriddedData(data, Metadata(name, units=units), [lat, lon, alt])
def _create_coord_list(self, filenames, data=None):
if data is None:
data = {} #initialise data dictionary
inData = netCDF4.Dataset(filenames[0]) #open netCDF file
data['longitude'] = np.array(
inData.variables['longitude']) #extract longitudes
data['latitude'] = np.array(
inData.variables['latitude']) #extract latitudes
origTimes = np.array(inData.variables['time']) #extract times
#convert to days since 1600-01-01 (cis col doesn't work otherwise - not sure why...):
niceDateTime = cf_units.num2date(origTimes,
'days since 1990-01-01 00:00:00',
'gregorian')
data['time'] = cf_units.date2num(niceDateTime,
'days since 1600-01-01 00:00:00',
'gregorian')
inData.close() #close netCDF file
coords = CoordList() #initialise coordinate list
#Append latitudes and longitudes to coordinate list:
coords.append(
Coord(
data['longitude'],
Metadata(name="longitude",
long_name='longitude',
standard_name='longitude',
shape=(len(data), ),
missing_value=-999.0,
units="degrees_east",
range=(-180, 180)), "x"))
coords.append(
Coord(
data['latitude'],
Metadata(name="latitude",
long_name='latitude',
standard_name='latitude',
shape=(len(data), ),
missing_value=-999.0,
units="degrees_north",
range=(-90, 90)), "y"))
coords.append(
Coord(
data['time'],
Metadata(name="time",
long_name='time',
standard_name='time',
shape=(len(data), ),
missing_value=-999.0,
units="days since 1600-01-01 00:00:00"), "t"))
return coords
def create_data_object(self, filenames, variable):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedData
var_data = read_many_files_individually(filenames, ["longitude", "latitude", "time", variable])
lon = Coord(var_data["longitude"], get_metadata(var_data["longitude"][0]), axis="x")
lat = Coord(var_data["latitude"], get_metadata(var_data["latitude"][0]), axis="y")
time = Coord(var_data["time"], get_metadata(var_data["time"][0]), axis="t")
coords = CoordList([lat, lon, time])
usr_var_data = var_data[variable]
return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), coords)
def _add_aux_coordinate(dim_coords, filename, aux_coord_name, length):
"""
Add an auxiliary coordinate to a list of (reshaped) dimension coordinates
:param dim_coords: CoordList of one-dimensional coordinates representing physical dimensions
:param filename: The data file containing the aux coord
:param aux_coord_name: The name of the aux coord to add to the coord list
:param length: The length of the data dimensions which this auxiliary coordinate should span
:return: A CoordList of reshaped (2D) physical coordinates plus the 2D auxiliary coordinate
"""
from cis.data_io.Coord import Coord
from cis.utils import expand_1d_to_2d_array
from cis.data_io.netcdf import read
# We assume that the auxilliary coordinate is the same shape across files
d = read(filename, [aux_coord_name])[aux_coord_name]
# Reshape to the length given
aux_data = expand_1d_to_2d_array(d[:], length, axis=0)
# Get the length of the auxiliary coordinate
len_y = d[:].size
for dim_coord in dim_coords:
dim_coord.data = expand_1d_to_2d_array(dim_coord.data, len_y, axis=1)
all_coords = dim_coords + [Coord(aux_data, get_metadata(d))]
return all_coords
def create_coords(self, filenames, variable=None):
"""
Reads the coordinates and data if required from the files
:param filenames: List of filenames to read coordinates from
:param variable: load a variable for the data
:return: Coordinates
"""
from cis.data_io.netcdf import read_many_files_individually
from cis.data_io.Coord import Coord, CoordList
from cis.exceptions import InvalidVariableError
variables = [("longitude", "x"), ("latitude", "y"), ("altitude", "z"), ("time", "t"), ("air_pressure", "p")]
dim_coords = CoordList()
for v in variables:
try:
var_data = read_many_files_individually(filenames, v[0])[v[0]]
dim_coords.append(Coord(var_data, get_metadata(var_data[0]), axis=v[1]))
except InvalidVariableError:
pass
if variable is None:
return UngriddedCoordinates(dim_coords)
else:
all_coords = self._add_aux_coordinate(dim_coords, filenames[0], 'DP_MID',
dim_coords.get_coord(standard_name='time').data.size)
usr_var_data = read_many_files_individually(filenames, variable)[variable]
return UngriddedData(usr_var_data, get_metadata(usr_var_data[0]), all_coords)
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
variables = [("longitude", "x"), ("latitude", "y")]
# if usr_variable is not None:
# variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(variables))
coords = CoordList()
var_data = read_many_files_individually(filenames,
[v[0] for v in variables])
for var, (name, axis) in zip(var_data.values(), variables):
try:
coords.append(Coord(var, get_metadata(var[0]), axis=axis))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
usr_var_data = read_many_files_individually(
filenames, usr_variable)[usr_variable]
res = UngriddedData(usr_var_data, get_metadata(usr_var_data[0]),
coords)
return res
def _create_fixed_value_coord(self, coord_axis, values, coord_units,
points_counts, coord_name):
"""
Create a coordinate with a fixed value
:param coord_axis: Axis of the coordinate in the coords
:param coord_name: The name of the coordinate
:param coord_units: The units for the coordinate
:param points_counts: Number of points for this coordinate, or list of sizes for multiple files
:param values: Value of coordinate, or list of values for multiple files
:return:
"""
from cis.data_io.Coord import Coord
values = listify(values)
points_counts = listify(points_counts)
all_points = np.array([])
# Create separate arrays with values and sizes corresponding to each of the different input files.
for value, points_count in zip(values, points_counts):
file_points = np.ma.array(np.zeros(points_count) + float(value))
all_points = np.append(all_points, file_points)
metadata = Metadata(name=coord_name,
shape=all_points.shape,
units=coord_units,
range=(min(values), max(values)))
return Coord(all_points, metadata, coord_axis)
def stack_data_list(data_list, var_name=None, units=None):
"""
Stacks a list of Ungridded data objects with the same data variable, but different coordinates into a single
UngriddedData object, with accompanying lat, lon and time data.
It assumes the coordinates have the same *sets* of coordinates, but that the coordinates themselves are different.
:param data_list: list of UngriddedData objects to be merged (stacked vertically)
:param string var_name: Name of final data variable
:param string units: Units of final data variable
:return: A merged UngriddedData object
"""
import numpy as np
from cis.data_io.ungridded_data import UngriddedData, Metadata
from cis.data_io.Coord import Coord
coords = []
all_data = np.hstack((d.data_flattened for d in data_list))
# Create mask (not all datasets might be masked)
mask = np.hstack((d.data.mask.ravel() if isinstance(
d.data, np.ma.MaskedArray) else np.zeros(d.shape, np.bool)
for d in data_list))
all_data = np.ma.MaskedArray(all_data, mask)
for c in data_list[0].coords():
coord_data = np.hstack((d.coord(c).data_flattened for d in data_list))
coords.append(Coord(coord_data, c.metadata, axis=c.axis))
return UngriddedData(
all_data,
Metadata(name=var_name or data_list[0].name(),
units=units or data_list[0].units), coords)
def _create_coord_list(self, filenames, variable=None):
import datetime as dt
from cis.time_util import convert_time_since_to_std_time, cis_standard_time_unit
from cis.utils import concatenate
from cf_units import Unit
from geotiepoints import modis5kmto1km
variables = ['Latitude', 'Longitude', 'View_time']
logging.info("Listing coordinates: " + str(variables))
sdata, vdata = hdf.read(filenames, variables)
apply_interpolation = False
if variable is not None:
scale = self.__get_data_scale(filenames[0], variable)
apply_interpolation = True if scale is "1km" else False
lat_data = hdf.read_data(sdata['Latitude'], _get_MODIS_SDS_data)
lat_metadata = hdf.read_metadata(sdata['Latitude'], "SD")
lon_data = hdf.read_data(sdata['Longitude'], _get_MODIS_SDS_data)
lon_metadata = hdf.read_metadata(sdata['Longitude'], "SD")
if apply_interpolation:
lon_data, lat_data = modis5kmto1km(lon_data, lat_data)
lat_coord = Coord(lat_data, lat_metadata, 'Y')
lon_coord = Coord(lon_data, lon_metadata, 'X')
time = sdata['View_time']
time_metadata = hdf.read_metadata(time, "SD")
# Ensure the standard name is set
time_metadata.standard_name = 'time'
time_metadata.units = cis_standard_time_unit
t_arrays = []
for f, d in zip(filenames, time):
time_start = self._get_start_date(f)
t_data = _get_MODIS_SDS_data(
d) / 24.0 # Convert hours since to days since
t_offset = time_start - dt.datetime(1600, 1,
1) # Convert to CIS time
t_arrays.append(t_data + t_offset.days)
time_coord = Coord(concatenate(t_arrays), time_metadata, "T")
return CoordList([lat_coord, lon_coord, time_coord])
def test_GIVEN_missing_coord_values_WHEN_coords_THEN_missing_values_removed(
self):
x_points = np.arange(-10, 11, 5)
y_points = np.arange(-5, 6, 5)
y, x = np.meshgrid(y_points, x_points)
y = np.ma.masked_array(y, np.zeros(y.shape, dtype=bool))
y.mask[1, 2] = True
x = Coord(x, Metadata(standard_name='latitude', units='degrees'))
y = Coord(y, Metadata(standard_name='longitude', units='degrees'))
coords = CoordList([x, y])
ug = UngriddedCoordinates(coords)
coords = ug.coords()
for coord in coords:
assert_that(len(coord.data), is_(14))
def test_GIVEN_missing_coord_values_WHEN_data_flattened_THEN_missing_values_removed(
self):
x_points = np.arange(-10, 11, 5)
y_points = np.arange(-5, 6, 5)
y, x = np.meshgrid(y_points, x_points)
y = np.ma.masked_array(y, np.zeros(y.shape, dtype=bool))
y.mask[1, 2] = True
x = Coord(x, Metadata(standard_name='latitude', units='degrees'))
y = Coord(y, Metadata(standard_name='longitude', units='degrees'))
coords = CoordList([x, y])
data = np.reshape(np.arange(15) + 1.0, (5, 3))
ug = UngriddedData(None, Metadata(), coords, lambda x: data)
data = ug.data_flattened
assert_that(len(data), is_(14))
def create_coords(self, filenames, usr_variable=None):
from cis.data_io.netcdf import read_many_files_individually, get_metadata, get_netcdf_file_variables
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import UngriddedCoordinates, UngriddedData
from cis.exceptions import InvalidVariableError
# We have to read it once first to find out which variables are in there. We assume the set of coordinates in
# all the files are the same
file_variables = get_netcdf_file_variables(filenames[0])
def get_axis_std_name(var):
axis=None
lvar = var.lower()
if lvar == 'longitude':
axis = 'x', 'longitude'
if lvar == 'latitude':
axis = 'y', 'latitude'
if lvar == 'G_ALT' or lvar == 'altitude' or lvar == 'pressure_altitude':
axis = 'z', 'altitude'
if lvar == 'time':
axis = 't', 'time'
if lvar == 'p' or lvar == 'pressure' or lvar == 'static_pressure':
axis = 'p', 'air_pressure'
return axis
all_coord_variables = [(v, get_axis_std_name(v)) for v in file_variables if get_axis_std_name(v) is not None]
# Get rid of any duplicates
coord_variables = []
for v in all_coord_variables:
if v is None or v[1][1] not in [x[1][1] for x in coord_variables]:
coord_variables.append(v)
all_variables = coord_variables.copy()
if usr_variable is not None:
all_variables.append((usr_variable, ''))
logging.info("Listing coordinates: " + str(all_variables))
coords = CoordList()
var_data = read_many_files_individually(filenames, [v[0] for v in all_variables])
for name, axis_std_name in coord_variables:
try:
meta = get_metadata(var_data[name][0])
if meta.standard_name is None:
meta.standard_name = axis_std_name[1]
coords.append(Coord(var_data[name], meta, axis=axis_std_name[0]))
except InvalidVariableError:
pass
# Note - We don't need to convert this time coord as it should have been written in our
# 'standard' time unit
if usr_variable is None:
res = UngriddedCoordinates(coords)
else:
res = UngriddedData(var_data[usr_variable], get_metadata(var_data[usr_variable][0]), coords)
return res
def _create_coord_list(self, filename):
import numpy as np
coords = CoordList()
time_data = read(filename, 'time')['time']
try:
alt_data = read(filename, 'altitude')['altitude']
except InvalidVariableError:
alt_data = read(filename, 'range')['range']
len_y = alt_data.shape[1]
time_arr = utils.expand_1d_to_2d_array(time_data[:], len_y, axis=1)
t_coord = Coord(time_arr, get_metadata(time_data), axis='x')
t_coord.convert_to_std_time()
coords.append(t_coord)
#alt_arr = utils.expand_1d_to_2d_array(alt_data[:], len_x, axis=0)
alt_arr = alt_data[:, :, 0] #eliminate "angle" axis
#alt_arr = alt_data #eliminate "angle" axis
coords.append(Coord(alt_arr, get_metadata(alt_data), axis='y'))
lat_data = read(filename, 'latitude')['latitude']
lat_arr = np.ones(alt_arr.shape) * lat_data[:]
coords.append(Coord(lat_arr, get_metadata(lat_data)))
lon_data = read(filename, 'longitude')['longitude']
lon_arr = np.ones(alt_arr.shape) * lon_data[:]
coords.append(Coord(lon_arr, get_metadata(lon_data)))
return coords
def _create_coord_list(self, filenames, data=None):
from cis.data_io.ungridded_data import Metadata
from cis.data_io.aeronet import load_multiple_aeronet
from cis.time_util import cis_standard_time_unit as ct
if data is None:
data = load_multiple_aeronet(filenames)
coords = CoordList()
coords.append(Coord(data['longitude'], Metadata(name="Longitude", shape=(len(data),),
units="degrees_east", range=(-180, 180))))
coords.append(Coord(data['latitude'], Metadata(name="Latitude", shape=(len(data),),
units="degrees_north", range=(-90, 90))))
coords.append(Coord(data['altitude'], Metadata(name="Altitude", shape=(len(data),), units="meters")))
coords.append(Coord(data["datetime"], Metadata(name="DateTime", standard_name='time', shape=(len(data),),
units=ct), "X"))
return coords
def can_append_to_list_of_coordinates():
list = create_dummy_coordinates_list()
list.append(
Coord(numpy.array([5, 4]),
Metadata(standard_name='altitude'),
axis='Z'))
assert (len(list) == 3)
assert (list[2].standard_name == 'altitude')
assert (list[2].axis == 'Z')
def _create_coord(self, coord_axis, data_variable_name, data_variables, standard_name):
"""
Create a coordinate for the co-ordinate list
:param coord_axis: axis of the coordinate in the coords
:param data_variable_name: the name of the variable in the data
:param data_variables: the data variables
:param standard_name: the standard name it should have
:return: a coords object
"""
from cis.data_io.Coord import Coord
coordinate_data_objects = []
for d in data_variables[data_variable_name]:
m = get_metadata(d)
m.standard_name = standard_name
coordinate_data_objects.append(Coord(d, m, coord_axis))
return Coord.from_many_coordinates(coordinate_data_objects)
def _create_coord_list(self, filenames, data=None):
from cis.data_io.ungridded_data import Metadata
from cis.data_io.aeronet import load_multiple_aeronet
if data is None:
data = load_multiple_aeronet(filenames)
coords = CoordList()
coords.append(Coord(data['longitude'], Metadata(name="Longitude", shape=(len(data),),
units="degrees_east", range=(-180, 180))))
coords.append(Coord(data['latitude'], Metadata(name="Latitude", shape=(len(data),),
units="degrees_north", range=(-90, 90))))
coords.append(Coord(data['altitude'], Metadata(name="Altitude", shape=(len(data),), units="meters")))
time_coord = Coord(data["datetime"], Metadata(name="DateTime", standard_name='time', shape=(len(data),),
units="DateTime Object"), "X")
time_coord.convert_datetime_to_standard_time()
coords.append(time_coord)
return coords
def _create_coord_list(self, filenames):
from cis.data_io.netcdf import read_many_files_individually, get_metadata
from cis.data_io.Coord import Coord
variables = ["lat", "lon", "time"]
logging.info("Listing coordinates: " + str(variables))
var_data = read_many_files_individually(filenames, variables)
coords = CoordList()
coords.append(Coord(var_data['lat'], get_metadata(var_data['lat'][0]), 'Y'))
coords.append(Coord(var_data['lon'], get_metadata(var_data['lon'][0]), 'X'))
time_coord = Coord(var_data['time'], get_metadata(var_data['time'][0]))
# TODO: Is this really julian?
time_coord.convert_julian_to_std_time()
coords.append(time_coord)
return coords
def _create_coord_list(self, filenames):
from cis.data_io.netcdf import read_many_files, get_metadata
from cis.data_io.Coord import Coord
import datetime
# FIXME: when reading an existing file variables might be "latitude", "longitude"
variables = ["lat", "lon", "time"]
logging.info("Listing coordinates: " + str(variables))
data = read_many_files(filenames, variables, dim="pixel_number")
coords = CoordList()
coords.append(Coord(data["lon"], get_metadata(data["lon"]), "X"))
coords.append(Coord(data["lat"], get_metadata(data["lat"]), "Y"))
time_coord = Coord(data["time"], get_metadata(data["time"]), "T")
time_coord.convert_TAI_time_to_std_time(datetime.datetime(1970, 1, 1))
coords.append(time_coord)
return coords
def _create_coord_list(self, filenames, variable=None):
import datetime as dt
variables = ['Latitude', 'Longitude', 'Scan_Start_Time']
logging.info("Listing coordinates: " + str(variables))
sdata, vdata = hdf.read(filenames, variables)
apply_interpolation = False
if variable is not None:
scale = self.__get_data_scale(filenames[0], variable)
apply_interpolation = True if scale is "1km" else False
lat = sdata['Latitude']
lat_data = hdf.read_data(lat, _get_MODIS_SDS_data)
lon = sdata['Longitude']
lon_data = hdf.read_data(lon, _get_MODIS_SDS_data)
if apply_interpolation:
lon_data, lat_data = modis5kmto1km(lon_data[:], lat_data[:])
lat_metadata = hdf.read_metadata(lat, "SD")
lat_coord = Coord(lat_data, lat_metadata, 'Y')
lon_metadata = hdf.read_metadata(lon, "SD")
lon_coord = Coord(lon_data, lon_metadata, 'X')
time = sdata['Scan_Start_Time']
time_metadata = hdf.read_metadata(time, "SD")
# Ensure the standard name is set
time_metadata.standard_name = 'time'
time_data = hdf.read_data(time, _get_MODIS_SDS_data)
time_data = np.repeat(np.repeat(time_data, 5, axis=0), 5, axis=1)
time_coord = Coord(time_data, time_metadata, "T")
time_coord.convert_TAI_time_to_std_time(dt.datetime(1993, 1, 1, 0, 0, 0))
return CoordList([lat_coord, lon_coord, time_coord])
def _create_coord(self, coord_axis, data_variable_name, data_variables, standard_name):
"""
Create a coordinate for the co-ordinate list
:param coord_axis: axis of the coordinate in the coords
:param data_variable_name: the name of the variable in the data
:param data_variables: the data variables
:param standard_name: the standard name it should have
:return: a coords object
"""
from cis.data_io.Coord import Coord
coordinate_data_objects = []
for d in data_variables[data_variable_name]:
m = get_metadata(d)
m.alter_standard_name(standard_name)
coordinate_data_objects.append(Coord(d, m, coord_axis))
return Coord.from_many_coordinates(coordinate_data_objects)
def _create_coord(self, coord_axis, data_variable_name, data_variables, standard_name):
"""
Create a coordinate for the co-ordinate list
:param coord_axis: axis of the coordinate in the coords
:param data_variable_name: the name of the variable in the data
:param data_variables: the data variables
:param standard_name: the standard name it should have
:return: a coords object
"""
from cis.data_io.netcdf import get_metadata
from cis.data_io.Coord import Coord
from cf_units import Unit
import logging
coordinate_data_objects = []
for d in data_variables[data_variable_name]:
data = get_data(d)
m = get_metadata(d)
m._name = m._name.lower()
m.standard_name = standard_name
if standard_name == 'air_pressure':
if not isinstance(m.units, Unit):
if ',' in m.units:
# Try splitting any commas out
m.units = m.units.split(',')[0]
if ' ' in m.units:
# Try splitting any spaces out
m.units = m.units.split()[0]
if str(m.units) == 'mb' or str(m.units) == 'Mb':
# Try converting to standard nomencleture
m.units = 'mbar'
if str(m.units) == 'hpa':
m.units = 'hPa'
logging.info("Parsed air pressure units {old}".format(old=m.units))
logging.info('Converting to hPa')
if not isinstance(m.units, str):
data = m.units.convert(data, 'hPa')
m.units = 'hPa'
coordinate_data_objects.append(Coord(data, m, coord_axis))
return Coord.from_many_coordinates(coordinate_data_objects)
def can_not_convert_time_without_since_in_units_but_no_timestamp():
times = numpy.array([0, 1])
units = "Days from the file reference point 1601-01-01"
coord = Coord(times, Metadata(units=units))
coord.convert_to_std_time()
