def _make_two_ungridded_data(self):
data1 = mock.make_regular_2d_ungridded_data_with_missing_values()
data2 = mock.make_regular_2d_ungridded_data_with_missing_values()
data1.metadata._name = 'var1'
data2.metadata._name = 'var2'
data1.filenames = ['filename1']
data2.filenames = ['filename2']
self.data = [data1, data2]
def test_horizontal_constraint_in_2d_with_missing_values(self):
ug_data = mock.make_regular_2d_ungridded_data_with_missing_values()
ug_data_points = ug_data.as_data_frame(time_index=False,
name='vals').dropna(axis=0)
coord_map = None
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
sample_points = None # Not used
index.index_data(sample_points, ug_data_points, coord_map, leafsize=2)
constraint.haversine_distance_kd_tree_index = index
for i, sample_point in ug_data.as_data_frame(time_index=False,
name='vals').iterrows():
new_points = constraint.constrain_points(sample_point,
ug_data_points)
new_vals = new_points.vals
if np.isnan(sample_point.vals):
ref_vals = np.array([])
else:
ref_vals = np.array([sample_point.vals])
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_can_access_point_in_ungridded_hyper_point_view(self):
ug = mock.make_regular_2d_ungridded_data_with_missing_values()
hpv = ug.get_all_points()
p = hpv[10]
assert(p.val[0] == 11.0)
assert(p.longitude == 0.0)
assert(p.latitude == 5.0)
def test_can_iterate_over_non_masked_points(self):
ug = mock.make_regular_2d_ungridded_data_with_missing_values()
hpv = ug.get_non_masked_points()
count = 0
vals = []
for p in hpv:
count += 1
vals.append(p.val[0])
assert(count == 12)
assert(vals == [1, 2, 3, 4, 6, 7, 8, 10, 11, 12, 14, 15])
def test_GIVEN_ungridded_missing_values_WHEN_calculate_THEN_missing_values_preserved(self):
data = mock.make_regular_2d_ungridded_data_with_missing_values()
data.metadata._name = 'var1'
self.data = [data]
expr = 'var1 + 10'
res = self.calc.evaluate(self.data, expr)
expected = numpy.ma.masked_invalid([[11, 12, 13], [14, float('Nan'), 16], [17, 18, float('Nan')],
[20, 21, 22], [float('Nan'), 24, 25]]).flatten()
compare_masked_arrays(res.data, expected)
def test_GIVEN_ungridded_data_with_missing_vals_WHEN_call_as_data_frame_THEN_returns_valid_data_frame(self):
from cis.test.util.mock import make_regular_2d_ungridded_data_with_missing_values
ug_data = make_regular_2d_ungridded_data_with_missing_values()
df = ug_data.as_data_frame()
assert_that(df['rainfall_rate'][5] == 6)
assert_that(df['latitude'][7] == 0)
assert_that(df['rainfall_rate'].median() == 7.5)
assert_that(np.isnan(df['rainfall_rate'][8]))
def test_GIVEN_ungridded_data_with_missing_vals_WHEN_call_as_data_frame_THEN_returns_valid_data_frame(
self):
from cis.test.util.mock import make_regular_2d_ungridded_data_with_missing_values
ug_data = make_regular_2d_ungridded_data_with_missing_values()
df = ug_data.as_data_frame()
assert_that(df['rainfall_rate'][5] == 6)
assert_that(df['latitude'][7] == 0)
assert_that(df['rainfall_rate'].median() == 7.5)
assert_that(np.isnan(df['rainfall_rate'][8]))
def test_can_modify_a_hyperpoint_value(self):
ug = mock.make_regular_2d_ungridded_data_with_missing_values()
hpv = ug.get_all_points()
hpv[6] = 99
p = hpv[6]
assert(p.latitude == 0)
assert(p.longitude == -5)
assert(p.val[0] == 99)
# Check an unmodified point.
p = hpv[5]
assert(p.latitude == -5)
assert(p.longitude == 5)
assert(p.val[0] == 6)
def test_can_enumerate_non_masked_points(self):
ug = mock.make_regular_2d_ungridded_data_with_missing_values()
hpv = ug.get_non_masked_points()
count = 0
indices = []
vals = []
for i, p in hpv.enumerate_non_masked_points():
count += 1
indices.append(i)
vals.append(p.val[0])
assert(count == 12)
assert(indices == [0, 1, 2, 3, 5, 6, 7, 9, 10, 11, 13, 14])
assert(vals == [1, 2, 3, 4, 6, 7, 8, 10, 11, 12, 14, 15])
def test_can_set_a_hyperpoint(self):
ug = mock.make_regular_2d_ungridded_data_with_missing_values()
hpv = ug.get_all_points()
p = hpv[6]
p_new = p.modified(lon=123, val=99)
hpv[6] = p_new
p = hpv[6]
assert(p.latitude == 0)
assert(p.longitude == 123)
assert(p.val[0] == 99)
# Check an unmodified point.
p = hpv[5]
assert(p.latitude == -5)
assert(p.longitude == 5)
assert(p.val[0] == 6)
def test_name_method(self):
from cis.test.util.mock import make_regular_2d_ungridded_data_with_missing_values
d = make_regular_2d_ungridded_data_with_missing_values()
# Standard name
assert d.name() == 'rainfall_rate'
d.standard_name = None
# Long name
assert d.name() == 'TOTAL RAINFALL RATE: LS+CONV KG/M2/S'
d.long_name = ''
# Var_name
assert d.name() == 'rain'
d.var_name = None
# Default
assert d.name() == 'unknown'
# User specified default
assert d.name('test') == 'test'
# Empty default
assert d.name('') == ''
def test_name_method(self):
from cis.test.util.mock import make_regular_2d_ungridded_data_with_missing_values
d = make_regular_2d_ungridded_data_with_missing_values()
# Standard name
assert d.name() == 'rainfall_rate'
d.standard_name = None
# Long name
assert d.name() == 'TOTAL RAINFALL RATE: LS+CONV KG/M2/S'
d.long_name = ''
# Var_name
assert d.name() == 'rain'
d.var_name = None
# Default
assert d.name() == 'unknown'
# User specified default
assert d.name('test') == 'test'
# Empty default
assert d.name('') == ''
def test_horizontal_constraint_in_2d_with_missing_values(self):
ug_data = mock.make_regular_2d_ungridded_data_with_missing_values()
ug_data_points = ug_data.as_data_frame(time_index=False, name='vals').dropna(axis=0)
coord_map = None
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
sample_points = None # Not used
index.index_data(sample_points, ug_data_points, coord_map, leafsize=2)
constraint.haversine_distance_kd_tree_index = index
for i, sample_point in ug_data.as_data_frame(time_index=False, name='vals').iterrows():
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
if np.isnan(sample_point.vals):
ref_vals = np.array([])
else:
ref_vals = np.array([sample_point.vals])
eq_(ref_vals.size, new_vals.size)
assert (np.equal(ref_vals, new_vals).all())
def test_horizontal_constraint_in_2d_with_missing_values(self):
ug_data = mock.make_regular_2d_ungridded_data_with_missing_values()
ug_data_points = ug_data.get_non_masked_points()
coord_map = None
# One degree near 0, 0 is about 110km in latitude and longitude, so 300km should keep us to within 3 degrees
# in each direction
constraint = SepConstraintKdtree(h_sep=400)
index = HaversineDistanceKDTreeIndex()
sample_points = None # Not used
index.index_data(sample_points, ug_data_points, coord_map, leafsize=2)
constraint.haversine_distance_kd_tree_index = index
for sample_point in ug_data.get_all_points():
new_points = constraint.constrain_points(sample_point, ug_data_points)
new_vals = new_points.vals
if sample_point.val[0] is np.ma.masked:
ref_vals = np.array([])
else:
ref_vals = np.array([sample_point.val])
eq_(ref_vals.size, new_vals.size)
assert np.equal(ref_vals, new_vals).all()
def test_get_coordinates_points_returns_points(self):
ug = make_regular_2d_ungridded_data_with_missing_values()
points = ug.get_coordinates_points()
num_points = len([p for p in points])
assert (num_points == 15)
def test_get_coordinates_points_returns_points(self):
ug = make_regular_2d_ungridded_data_with_missing_values()
points = ug.get_coordinates_points()
num_points = len([p for p in points])
assert(num_points == 15)
def test_can_create_ungridded_hyper_point_view(self):
ug = mock.make_regular_2d_ungridded_data_with_missing_values()
all_coords = ug._coords.find_standard_coords()
flattened_coords = [(c.data_flattened if c is not None else None) for c in all_coords]
hpv = UngriddedHyperPointView(flattened_coords, ug.data_flattened)
assert(len(hpv) == 15)
def test_accessing_with_invalid_index_raises_error(self):
ug = mock.make_regular_2d_ungridded_data_with_missing_values()
hpv = ug.get_all_points()
p = hpv[16]
