def _create_coord_list(self):
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import Metadata
from cis.time_util import cis_standard_time_unit as cstu
# These implement a lot of what is necessary, but aren't in CIS style
from acp_utils import rolling_window
from orbit import ATSR
lat_data = []
lon_data = []
time_data = []
for fname in self.filenames:
prod = ATSR(fname)
lat_data.append(prod.lat)
lon_data.append(prod.lon)
time_data.append(prod.get_time())
# TODO: Properly define metadata
lat_meta = Metadata(standard_name="latitude", units="degrees")
lon_meta = Metadata(standard_name="longitude", units="degrees")
time_meta = Metadata(standard_name="time", units=cstu)
lat = Coord(concatenate(lat_data), lat_meta, "Y")
lat.update_shape()
lat.update_range()
lon = Coord(concatenate(lon_data), lon_meta, "Y")
lon.update_shape()
lon.update_range()
time = Coord(concatenate(time_data), time_meta, "T")
time.update_shape()
time.update_range()
return CoordList([lat, lon, time])
def setUp(self):
x_points = np.arange(-10, 11, 5)
y_points = np.arange(-5, 6, 5)
y, x = np.meshgrid(y_points, x_points)
x = Coord(
x, Metadata(name='lat', standard_name='latitude', units='degrees'))
y = Coord(
y, Metadata(name='lon', standard_name='longitude',
units='degrees'))
data = np.reshape(np.arange(15) + 1.0, (5, 3))
self.coords = CoordList([x, y])
ug1 = UngriddedData(
data,
Metadata(standard_name='rainfall_flux',
long_name="TOTAL RAINFALL RATE: LS+CONV KG/M2/S",
units="kg m-2 s-1",
missing_value=-999), self.coords)
ug2 = UngriddedData(
data * 0.1,
Metadata(standard_name='snowfall_flux',
long_name="TOTAL SNOWFALL RATE: LS+CONV KG/M2/S",
units="kg m-2 s-1",
missing_value=-999), self.coords)
self.ungridded_data_list = UngriddedDataList([ug1, ug2])
def create_dummy_coordinates_list():
coord1 = Coord(numpy.array([5, 4]),
Metadata(standard_name='grid_latitude'),
axis='Y')
coord2 = Coord(numpy.array([5, 4]),
Metadata(standard_name='grid_longitude'),
axis='X')
return CoordList([coord1, coord2])
def _create_coord_list(self):
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import Metadata
from cis.time_util import convert_sec_since_to_std_time
from os.path import basename
lat_all = []
lon_all = []
time_all = []
for fname in self.filenames:
var_name = self.gdal_variable_name(fname, "Optical_Depth_055")
if self.grid_path:
granule = basename(fname).split(".")[2]
lat_data, lon_data = self._read_grid_centres(granule)
else:
lat_data, lon_data = self._calculate_grid_centres(var_name)
time_data = self._calculate_grid_time(var_name, lat_data, lon_data)
# Workaround files containing only one day
sh = (-1, ) + lat_data.shape
time_data = time_data.reshape(sh)
keep = np.logical_not(self._read_qcmask(fname)).reshape(sh)
for time_slice, keep_slice in zip(time_data, keep):
lat_all.extend(lat_data[keep_slice])
lon_all.extend(lon_data[keep_slice])
time_all.extend(time_slice[keep_slice])
if len(lat_all) == 0:
raise NotImplementedError("It's empty!")
lat = Coord(
np.ma.array(lat_all),
Metadata(name="lat",
standard_name="latitude",
units="degrees",
range=(-90., 90.)), "Y")
lat.update_shape()
lon = Coord(
np.ma.array(lon_all),
Metadata(name="lon",
standard_name="longitude",
units="degrees",
range=(-180., 180.)), "X")
lon.update_shape()
time = Coord(
np.ma.array(time_all),
Metadata(name="time",
standard_name="time",
units="Seconds since 1993-1-1 00:00:00.0 0"), "T")
time.convert_TAI_time_to_std_time(MODIS_REFERENCE_TIME)
time.update_shape()
# Set the QC mask as we now know how many points we have
self._qcmask = np.full(lat.shape, False)
return CoordList([lat, lon, time])
def test_order_of_coords_doesnt_matter(self):
from iris.cube import Cube
from iris.coords import DimCoord
import numpy as np
from cis.data_io.gridded_data import make_from_cube
from cis.data_io.ungridded_data import UngriddedCoordinates, Metadata
from cis.data_io.Coord import Coord
cube_lat = DimCoord(np.linspace(-90, 90, 18),
standard_name='latitude',
units='degrees')
cube_lon = DimCoord(np.linspace(0, 359, 36),
standard_name='longitude',
units='degrees',
circular=True)
cube_alt = DimCoord(np.linspace(0, 10000, 10),
standard_name='altitude',
units='meters')
times = np.linspace(0, 30, 12)
cube_time = DimCoord(times,
standard_name='time',
units='days since 1970-01-01 00:00:00')
data = np.arange(12 * 18 * 36 * 10).reshape(12, 18, 36, 10)
source = make_from_cube(
Cube(data,
dim_coords_and_dims=[(cube_time, 0), (cube_lat, 1),
(cube_lon, 2), (cube_alt, 3)]))
n = 10
sample_lats = np.linspace(-9.1, 9.9, n)
sample_longs = np.linspace(-9.1, 9.9, n)
sample_alts = np.linspace(99, 599, n)
sample_times = np.linspace(0, 30, n)
sample = UngriddedCoordinates([
Coord(sample_lats, Metadata('latitude')),
Coord(sample_longs, Metadata('longitude')),
Coord(sample_alts, Metadata('altitude')),
Coord(sample_times,
Metadata('time', units='days since 1970-01-01 00:00:00'))
])
col = GriddedUngriddedCollocator()
output = col.collocate(sample, source, None, 'nn')[0]
source.transpose()
col = GriddedUngriddedCollocator()
assert_equal(
col.collocate(sample, source, None, 'nn')[0].data, output.data)
source.transpose((2, 1, 0, 3))
col = GriddedUngriddedCollocator()
assert_equal(
col.collocate(sample, source, None, 'nn')[0].data, output.data)
def setUp(self):
x_points = np.arange(-10, 11, 5)
y_points = np.arange(-5, 6, 5)
y, x = np.meshgrid(y_points, x_points)
self.x = Coord(x, Metadata(standard_name='latitude', units='degrees'))
self.y = Coord(y, Metadata(standard_name='longitude', units='degrees'))
coords = CoordList([self.x, self.y])
self.ug = UngriddedCoordinates(coords)
def get_file_metadata(filename, variable='', shape=None):
file = open(filename)
from cis.data_io.ungridded_data import Metadata
if variable is None:
variable = ''
metadata = Metadata(name=variable, long_name=variable, shape=shape)
lines = []
for i in range(0, 4):
lines.append(file.readline().replace("\n", "").split(","))
metadata.misc = lines
return metadata
def get_file_metadata(filename, variable='', shape=None):
file = open(filename)
from cis.data_io.ungridded_data import Metadata
if variable is None:
variable = ''
metadata = Metadata(name=variable, long_name=variable, shape=shape)
lines = []
for i in range(0, 4):
lines.append(file.readline().replace("\n", "").split(","))
metadata.misc = lines
return metadata
def can_find_many_coords_from_a_list_of_coordinates():
list = create_dummy_coordinates_list()
list.append(Coord(numpy.array([5, 4]), Metadata(name='testZ'), axis='Z'))
list.append(
Coord(numpy.array([5, 4]),
Metadata(name='testZ', standard_name='air_pressure')))
assert (len(list) == 4)
coords = list.get_coords(var_name='testZ')
assert (len(coords) == 2)
assert (coords[0].axis == 'Z')
assert (coords[1].axis == '')
assert (coords[1].name() == 'air_pressure')
def scene_to_cis(scn):
from cis.data_io.Coord import Coord
from cis.data_io.ungridded_data import Metadata, UngriddedData
data = scn['M05']
area = data.info['area']
units = data.info['units']
lats = Coord(
area.lats, Metadata(standard_name='latitude', units='degrees'), 'y')
lons = Coord(
area.lons, Metadata(standard_name='longitude', units='degrees'), 'x')
print(area.lons.info)
ug = UngriddedData(data, Metadata(name=data.info['name'], units=units), [lats, lons])
return ug
def get_file_metadata(filename, variable='', shape=None):
file = open(filename)
from cis.data_io.ungridded_data import Metadata
if variable is None:
variable = ''
version = get_aeronet_version(filename)
metadata = Metadata(name=variable, long_name=variable, shape=shape)
lines = []
for i in range(0, AERONET_HEADER_LENGTH[version]-1):
lines.append(file.readline().replace("\n", "").split(","))
metadata.misc = lines
return metadata
def make_mock_CALIOP_data(data, name='', units=''):
from cis.utils import expand_1d_to_2d_array
vertical_levels, swath_length = data.shape
lat = Coord(
expand_1d_to_2d_array(np.arange(swath_length), vertical_levels),
Metadata(standard_name='latitude'))
lon = Coord(
expand_1d_to_2d_array(np.arange(swath_length), vertical_levels),
Metadata(standard_name='longitude'))
alt = Coord(
expand_1d_to_2d_array(np.arange(vertical_levels), swath_length,
axis=1), Metadata(standard_name='altitude'))
return UngriddedData(data, Metadata(name, units=units), [lat, lon, alt])
def _create_bounded_coord_list(self):
from cis.data_io.Coord import Coord, CoordList
from cis.data_io.ungridded_data import Metadata
from cis.time_util import cis_standard_time_unit as cstu
# These implement a lot of what is necessary, but aren't in CIS style
from acp_utils import rolling_window
from orbit import MODIS
lat_data = []
lat_bounds = []
lon_data = []
lon_bounds = []
time_data = []
time_bounds = []
for fname in self.filenames:
prod = MODIS(fname)
lat_data.append(prod.lat)
lon_data.append(prod.lon)
lat_c = rolling_window(prod.lat_corner, (2, 2))
lat_bounds.append(lat_c.reshape(prod.shape + (4, )))
lon_c = rolling_window(prod.lon_corner, (2, 2))
lon_bounds.append(lon_c.reshape(prod.shape + (4, )))
t = prod.get_time()
time_data.append(t)
b = np.stack([t, np.roll(t, -1)], axis=2)
b[-1, :, 1] = 2 * t[-1, :] - t[-2, :]
time_bounds.append(b)
# TODO: Properly define metadata
lat_meta = Metadata(standard_name="latitude", units="degrees")
lon_meta = Metadata(standard_name="longitude", units="degrees")
time_meta = Metadata(standard_name="time", units=cstu)
lat = Coord(concatenate(lat_data), lat_meta, "Y")
lat.update_shape()
lat.update_range()
lat.bounds = concatenate(lat_bounds).reshape(lat.shape + (4, ))
lat.bounds[..., 2:4] = lat.bounds[..., [3, 2]]
lon = Coord(concatenate(lon_data), lon_meta, "Y")
lon.update_shape()
lon.update_range()
lon.bounds = concatenate(lon_bounds).reshape(lon.shape + (4, ))
lon.bounds[..., 2:4] = lon.bounds[..., [3, 2]]
time = Coord(concatenate(time_data), time_meta, "T")
time.update_shape()
time.update_range()
time.bounds = concatenate(time_bounds)
return CoordList([lat, lon, time])
def _create_coord_list(self, filenames, data=None):
if data is None:
data = {} #initialise data dictionary
inData = netCDF4.Dataset(filenames[0]) #open netCDF file
data['longitude'] = np.array(
inData.variables['longitude']) #extract longitudes
data['latitude'] = np.array(
inData.variables['latitude']) #extract latitudes
origTimes = np.array(inData.variables['time']) #extract times
#convert to days since 1600-01-01 (cis col doesn't work otherwise - not sure why...):
niceDateTime = cf_units.num2date(origTimes,
'days since 1990-01-01 00:00:00',
'gregorian')
data['time'] = cf_units.date2num(niceDateTime,
'days since 1600-01-01 00:00:00',
'gregorian')
inData.close() #close netCDF file
coords = CoordList() #initialise coordinate list
#Append latitudes and longitudes to coordinate list:
coords.append(
Coord(
data['longitude'],
Metadata(name="longitude",
long_name='longitude',
standard_name='longitude',
shape=(len(data), ),
missing_value=-999.0,
units="degrees_east",
range=(-180, 180)), "x"))
coords.append(
Coord(
data['latitude'],
Metadata(name="latitude",
long_name='latitude',
standard_name='latitude',
shape=(len(data), ),
missing_value=-999.0,
units="degrees_north",
range=(-90, 90)), "y"))
coords.append(
Coord(
data['time'],
Metadata(name="time",
long_name='time',
standard_name='time',
shape=(len(data), ),
missing_value=-999.0,
units="days since 1600-01-01 00:00:00"), "t"))
return coords
def can_not_convert_time_without_since_in_units_with_no_units():
times = numpy.array([0, 1])
units = ""
time_stamp_info = "1601-01-01"
coord = Coord(times, Metadata(units=units))
coord.convert_to_std_time(time_stamp_info)
def stack_data_list(data_list, var_name=None, units=None):
"""
Stacks a list of Ungridded data objects with the same data variable, but different coordinates into a single
UngriddedData object, with accompanying lat, lon and time data.
It assumes the coordinates have the same *sets* of coordinates, but that the coordinates themselves are different.
:param data_list: list of UngriddedData objects to be merged (stacked vertically)
:param string var_name: Name of final data variable
:param string units: Units of final data variable
:return: A merged UngriddedData object
"""
import numpy as np
from cis.data_io.ungridded_data import UngriddedData, Metadata
from cis.data_io.Coord import Coord
coords = []
all_data = np.hstack((d.data_flattened for d in data_list))
# Create mask (not all datasets might be masked)
mask = np.hstack((d.data.mask.ravel() if isinstance(
d.data, np.ma.MaskedArray) else np.zeros(d.shape, np.bool)
for d in data_list))
all_data = np.ma.MaskedArray(all_data, mask)
for c in data_list[0].coords():
coord_data = np.hstack((d.coord(c).data_flattened for d in data_list))
coords.append(Coord(coord_data, c.metadata, axis=c.axis))
return UngriddedData(
all_data,
Metadata(name=var_name or data_list[0].name(),
units=units or data_list[0].units), coords)
def create_data_object(self, filenames, variable):
from cis.exceptions import InvalidVariableError, CISError
import numpy as np
try:
data_obj = load_multiple_hysplit(filenames, [variable])
except ValueError:
raise InvalidVariableError(variable + " does not exist in " +
str(filenames))
except EOFError as e:
raise CISError(e)
except IOError as e:
raise CISError(e) # TODO
coords = self._create_coord_list(filenames, data_obj)
# WRITE STANDARD NAME GUESSER HERE
if variable == "PRESSURE":
variable = "air_pressure"
elif variable == "RELHUMID":
variable = "relative_humidity"
objM = Metadata(name=variable,
standard_name=variable,
long_name=variable,
shape=(len(data_obj[variable]), ),
missing_value=-99999.0)
#objM.standard_name = None
#print((len(data_obj[variable]),))
return UngriddedData(data_obj[variable], objM, coords)
def SolZen(self):
from orbit import ATSR
from cis.data_io.ungridded_data import Metadata
tmp = []
for f in self.filenames:
orbit = ATSR(f)
# Get tie point grid
sph = orbit._prod.get_sph()
tie_field = sph.get_field("VIEW_ANGLE_TIE_POINTS")
tie_pts = tie_field.get_elems()
# Get tie point values
scan_y = orbit._read_field("NADIR_VIEW_SOLAR_ANGLES_ADS",
"img_scan_y")
tie_solelv = orbit._read_field("NADIR_VIEW_SOLAR_ANGLES_ADS",
"tie_pt_sol_elev")
# Swath grid
x = np.arange(512) - 255.5
y = orbit._read_field("11500_12500_NM_NADIR_TOA_MDS", "img_scan_y")
y[:-1] += 0.5 * (y[1:] - y[:-1])
y[-1] += 0.5 * (y[-1] - y[-2])
solelv = orbit.extrap_atsr_angle(tie_pts, scan_y, x, y, tie_solelv)
tmp.append(90. - solelv)
return concatenate(tmp), Metadata(standard_name="solar_zenith_angle",
units="degrees")
def create_data_object(self, filenames, variable):
data_dict = {} #initialise data dictionary
inData = netCDF4.Dataset(filenames[0]) #open netCDF file
data_dict['longitude'] = np.array(
inData.variables['lon']) #extract longitudes
data_dict['latitude'] = np.array(
inData.variables['lat']) #extract latitudes
origTimes = np.array(inData.variables['time_counter']) #extract times
#Convert time to days since
niceDateTime = cf_units.num2date(origTimes,
'seconds since 1999-01-01 00:00:00',
'gregorian')
data_dict['time'] = cf_units.date2num(
niceDateTime, 'days since 1600-01-01 00:00:00', 'gregorian')
data_dict[variable] = np.array(
inData.variables[variable]) #extract requested variable
inData.close() #close netCDF file
coords = self._create_coord_list(filenames, data_dict)
return UngriddedData(
data_dict[variable],
Metadata(name=variable,
long_name=variable,
shape=(len(data_dict), ),
missing_value=-999.0,
units="1"), coords)
def test_GIVEN_multiple_ungridded_data_with_missing_data_WHEN_call_as_data_frame_THEN_returns_valid_data_frame(
self):
d = np.reshape(np.arange(15) + 10.0, (5, 3))
data = np.ma.masked_array(d, np.zeros(d.shape, dtype=bool))
data.mask[1, 2] = True
ug3 = UngriddedData(
data,
Metadata(name='hail',
long_name="TOTAL HAIL RATE: LS+CONV KG/M2/S",
units="kg m-2 s-1",
missing_value=-999), self.coords)
self.ungridded_data_list.append(ug3)
df = self.ungridded_data_list.as_data_frame()
assert_that(df['rainfall_flux'][5] == 6)
assert_almost_equal(df['snowfall_flux'][5], 0.6)
assert_that(df['latitude'][13] == 10)
assert_that(df['longitude'][0] == -5)
assert_almost_equal(df['TOTAL HAIL RATE: LS+CONV KG/M2/S'][1], 11.0)
assert_that(
np.isnan(
df['TOTAL HAIL RATE: LS+CONV KG/M2/S'][np.ravel_multi_index(
[1, 2], (5, 3))]))
self.ungridded_data_list.pop()
def _create_fixed_value_coord(self, coord_axis, values, coord_units,
points_counts, coord_name):
"""
Create a coordinate with a fixed value
:param coord_axis: Axis of the coordinate in the coords
:param coord_name: The name of the coordinate
:param coord_units: The units for the coordinate
:param points_counts: Number of points for this coordinate, or list of sizes for multiple files
:param values: Value of coordinate, or list of values for multiple files
:return:
"""
from cis.data_io.Coord import Coord
values = listify(values)
points_counts = listify(points_counts)
all_points = np.array([])
# Create separate arrays with values and sizes corresponding to each of the different input files.
for value, points_count in zip(values, points_counts):
file_points = np.ma.array(np.zeros(points_count) + float(value))
all_points = np.append(all_points, file_points)
metadata = Metadata(name=coord_name,
shape=all_points.shape,
units=coord_units,
range=(min(values), max(values)))
return Coord(all_points, metadata, coord_axis)
def can_convert_time_with_since_in_units():
times = numpy.array([0, 1])
units = "days since 1601-01-01"
coord = Coord(times, Metadata(units=units))
coord.convert_to_std_time()
assert_that(coord.data_flattened[0], is_(366.0), "time point")
def test_GIVEN_missing_coord_values_WHEN_coords_THEN_missing_values_removed(
self):
x_points = np.arange(-10, 11, 5)
y_points = np.arange(-5, 6, 5)
y, x = np.meshgrid(y_points, x_points)
y = np.ma.masked_array(y, np.zeros(y.shape, dtype=bool))
y.mask[1, 2] = True
x = Coord(x, Metadata(standard_name='latitude', units='degrees'))
y = Coord(y, Metadata(standard_name='longitude', units='degrees'))
coords = CoordList([x, y])
ug = UngriddedCoordinates(coords)
coords = ug.coords()
for coord in coords:
assert_that(len(coord.data), is_(14))
def test_GIVEN_missing_coord_values_WHEN_data_flattened_THEN_missing_values_removed(
self):
x_points = np.arange(-10, 11, 5)
y_points = np.arange(-5, 6, 5)
y, x = np.meshgrid(y_points, x_points)
y = np.ma.masked_array(y, np.zeros(y.shape, dtype=bool))
y.mask[1, 2] = True
x = Coord(x, Metadata(standard_name='latitude', units='degrees'))
y = Coord(y, Metadata(standard_name='longitude', units='degrees'))
coords = CoordList([x, y])
data = np.reshape(np.arange(15) + 1.0, (5, 3))
ug = UngriddedData(None, Metadata(), coords, lambda x: data)
data = ug.data_flattened
assert_that(len(data), is_(14))
def can_append_to_list_of_coordinates():
list = create_dummy_coordinates_list()
list.append(
Coord(numpy.array([5, 4]),
Metadata(standard_name='altitude'),
axis='Z'))
assert (len(list) == 3)
assert (list[2].standard_name == 'altitude')
assert (list[2].axis == 'Z')
def _create_coord_list(self, filenames, data=None):
from cis.data_io.ungridded_data import Metadata
from cis.data_io.aeronet import load_multiple_aeronet
from cis.time_util import cis_standard_time_unit as ct
if data is None:
data = load_multiple_aeronet(filenames)
coords = CoordList()
coords.append(Coord(data['longitude'], Metadata(name="Longitude", shape=(len(data),),
units="degrees_east", range=(-180, 180))))
coords.append(Coord(data['latitude'], Metadata(name="Latitude", shape=(len(data),),
units="degrees_north", range=(-90, 90))))
coords.append(Coord(data['altitude'], Metadata(name="Altitude", shape=(len(data),), units="meters")))
coords.append(Coord(data["datetime"], Metadata(name="DateTime", standard_name='time', shape=(len(data),),
units=ct), "X"))
return coords
def can_convert_time_without_since_in_units():
times = numpy.array([0, 1])
units = "Days from the file reference point 1601-01-01"
time_stamp_info = "1601-01-01"
coord = Coord(times, Metadata(units=units))
coord.convert_to_std_time(time_stamp_info)
assert_that(coord.data_flattened[0], is_(366.0), "time point")
def collocate(self, points, data, constraint, kernel):
"""
This collocator takes a list of HyperPoints and a data object (currently either Ungridded
data or a Cube) and returns one new LazyData object with the values as determined by the
constraint and kernel objects. The metadata for the output LazyData object is copied from
the input data object.
:param UngriddedData or UngriddedCoordinates points: Objects defining the sample points
:param GriddedData or GriddedDataList data: Data to resample
:param constraint: An instance of a Constraint subclass which takes a data object and
returns a subset of that data based on it's internal parameters
:param kernel: An instance of a Kernel subclass which takes a number of points and returns
a single value
:return: A single LazyData object
"""
from cis.collocation.gridded_interpolation import GriddedUngriddedInterpolator
log_memory_profile("GriddedUngriddedCollocator Initial")
if isinstance(data, list):
# Indexing and constraints (for SepConstraintKdTree) will only take place on the first iteration,
# so we really can just call this method recursively if we've got a list of data.
output = UngriddedDataList()
for var in data:
output.extend(self.collocate(points, var, constraint, kernel))
return output
if constraint is not None and not isinstance(constraint, DummyConstraint):
raise ValueError("A constraint cannot be specified for the GriddedUngriddedCollocator")
data_points = data
# First fix the sample points so that they all fall within the same 360 degree longitude range
_fix_longitude_range(points.coords(), points)
# Then fix the data points so that they fall onto the same 360 degree longitude range as the sample points
_fix_longitude_range(points.coords(), data_points)
log_memory_profile("GriddedUngriddedCollocator after data retrieval")
logging.info("--> Collocating...")
logging.info(" {} sample points".format(points.size))
if self.interpolator is None:
# Cache the interpolator
self.interpolator = GriddedUngriddedInterpolator(data, points, kernel, self.missing_data_for_missing_sample)
values = self.interpolator(data, fill_value=self.fill_value, extrapolate=self.extrapolate)
log_memory_profile("GriddedUngriddedCollocator after running kernel on sample points")
metadata = Metadata(self.var_name or data.var_name, long_name=self.var_long_name or data.long_name,
shape=values.shape, missing_value=self.fill_value, units=self.var_units or data.units)
set_standard_name_if_valid(metadata, data.standard_name)
return_data = UngriddedDataList([UngriddedData(values, metadata, points.coords())])
log_memory_profile("GriddedUngriddedCollocator final")
return return_data
def create_coords(self, filenames, variable=None):
from cis.data_io.ungridded_data import Metadata
from numpy import genfromtxt, NaN
from cis.exceptions import InvalidVariableError
from cis.time_util import convert_datetime_to_std_time
import dateutil.parser as du
array_list = []
for filename in filenames:
try:
array_list.append(genfromtxt(filename, dtype="f8,f8,f8,O,f8",
names=['latitude', 'longitude', 'altitude', 'time', 'value'],
delimiter=',', missing_values='', usemask=True, invalid_raise=True,
converters={"time": du.parse}))
except:
raise IOError('Unable to read file ' + filename)
data_array = utils.concatenate(array_list)
n_elements = len(data_array['latitude'])
coords = CoordList()
coords.append(Coord(data_array["latitude"],
Metadata(standard_name="latitude", shape=(n_elements,), units="degrees_north")))
coords.append(Coord(data_array["longitude"],
Metadata(standard_name="longitude", shape=(n_elements,), units="degrees_east")))
coords.append(
Coord(data_array["altitude"], Metadata(standard_name="altitude", shape=(n_elements,), units="meters")))
time_arr = convert_datetime_to_std_time(data_array["time"])
time = Coord(time_arr,
Metadata(standard_name="time", shape=(n_elements,), units="days since 1600-01-01 00:00:00"))
coords.append(time)
if variable:
try:
data = UngriddedData(data_array['value'], Metadata(name="value", shape=(n_elements,), units="unknown",
missing_value=NaN), coords)
except:
InvalidVariableError("Value column does not exist in file " + filenames)
return data
else:
return UngriddedCoordinates(coords)
def _create_coord_list(self, filenames, data=None):
from cis.data_io.ungridded_data import Metadata
from cis.time_util import cis_standard_time_unit as ct
import numpy as np
if data is None:
data = load_multiple_hysplit(filenames) # TODO error handling
coords = CoordList()
#print(data['DATETIMES'])
latM = Metadata(standard_name="latitude",
shape=(len(data['LAT']), ),
units="degrees_north",
range=(-90, 90))
lonM = Metadata(standard_name="longitude",
shape=(len(data['LON']), ),
units="degrees_east",
range=(-180, 180))
altM = Metadata(standard_name="altitude",
shape=(len(data['ALT']), ),
units="m")
timeM = Metadata(standard_name="time",
shape=(len(data['DATETIMES']), ),
units=str(ct))
#timeM = Metadata(name="DateTime", standard_name="time", shape=(len(data['DATETIMES']),), units=str(ct))
pressM = Metadata(standard_name="air_pressure",
shape=(len(data['PRESSURE']), ),
units="Pa")
#start_timeM = Metadata(name="start_time", standard_name="forecast_reference_time", shape=(len(data['STARTING_TIME']),), units=str(ct))
#start_heightM = Metadata(name="start_height", shape=(len(data['STARTING_HEIGHT']),), units="meters")
#station_noM = Metadata(name="station_no", standard_name="institution", shape=(len(data['STATION_NO']),))
coords.append(Coord(data['DATETIMES'], timeM))
coords.append(Coord(data['PRESSURE'], pressM))
coords.append(Coord(data['LAT'], latM))
coords.append(Coord(data['LON'], lonM))
coords.append(Coord(data['ALT'], altM))
#coords.append(Coord(data['STARTING_TIME'], start_timeM))
#coords.append(Coord(data['STARTING_HEIGHT'], start_heightM))
#coords.append(Coord(data['STATION_NO'], station_noM))
return coords
def create_data_object(self, filenames, variable):
logging.info("gathering data for variable " + str(variable))
for filename in filenames:
data = []
logging.info("gatherings metadata for variable " + str(variable))
metadata = Metadata()
coords = self.create_coords(filenames)
return UngriddedData(data, metadata, coords)
def test_unit_setter(self):
# Standard units
m = Metadata(units='kg m-2 s-1')
assert m.units == Unit('kg m-2 s-1')
# Case sensitive units
m.units = 'hPa'
assert m.units == Unit('hPa')
# Units with number in
m.units = '#/cm3'
assert m.units == Unit('cm-3')
# standard time since units
m.units = 'days since 1999-01-01'
assert m.units == Unit('days since 1999-01-01')
# Capitalised time since
m.units = 'Days since 1999-01-01'
assert m.units == Unit('days since 1999-01-01')
# Time since with a colon and capital S
m.units = 'hours Since: 1999-01-01'
assert m.units == Unit('hours Since 1999-01-01')