def test_read_not_empty_ascatb(self):
test_file = path.join(path.dirname(__file__), 'datafiles',
'not_empty_ascatb.nc4')
swath_reader = processors.SwathReadingProcessor('wind_speed',
'lat',
'lon',
time='time')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "NUMROWS:0:1,NUMCELLS:0:82"
input_tile.summary.CopyFrom(tile_summary)
results = list(swath_reader.process(input_tile))
self.assertEqual(1, len(results))
for nexus_tile in results:
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
self.assertEqual(0, len(nexus_tile.tile.swath_tile.meta_data))
tile = nexus_tile.tile.swath_tile
self.assertEqual(82, from_shaped_array(tile.latitude).size)
self.assertEqual(82, from_shaped_array(tile.longitude).size)
tile1_data = np.ma.masked_invalid(
from_shaped_array(results[0].tile.swath_tile.variable_data))
self.assertEqual((1, 82), tile1_data.shape)
self.assertEqual(82, np.ma.count(tile1_data))
def test_read_empty_mur(self):
test_file = path.join(path.dirname(__file__), 'datafiles',
'empty_mur.nc4')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "time:0:1,lat:0:10,lon:0:10"
input_tile.summary.CopyFrom(tile_summary)
results = list(self.module.process(input_tile))
self.assertEqual(1, len(results))
for nexus_tile in results:
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('grid_tile'))
tile = nexus_tile.tile.grid_tile
self.assertEqual(10, len(from_shaped_array(tile.latitude)))
self.assertEqual(10, len(from_shaped_array(tile.longitude)))
the_data = np.ma.masked_invalid(
from_shaped_array(tile.variable_data))
self.assertEqual((1, 10, 10), the_data.shape)
self.assertEqual(0, np.ma.count(the_data))
self.assertTrue(tile.HasField('time'))
def test_kelvin_to_celsius(self):
test_file = path.join(path.dirname(__file__), 'dumped_nexustiles',
'avhrr_nonempty_nexustile.bin')
with open(test_file, 'rb') as f:
nexustile_str = f.read()
nexus_tile_before = nexusproto.NexusTile.FromString(nexustile_str)
sst_before = from_shaped_array(
nexus_tile_before.tile.grid_tile.variable_data)
results = list(self.module.process(nexustile_str))
self.assertEqual(1, len(results))
nexus_tile_after = results[0]
sst_after = from_shaped_array(
nexus_tile_after.tile.grid_tile.variable_data)
# Just spot check a couple of values
expected_sst = np.subtract(sst_before[0][0][0], np.float32(273.15))
self.assertEqual(expected_sst, sst_after[0][0][0])
expected_sst = np.subtract(sst_before[0][9][9], np.float32(273.15))
self.assertEqual(expected_sst, sst_after[0][9][9])
def test_read_empty_mur(self):
test_file = path.join(path.dirname(__file__), 'datafiles',
'empty_mur.nc4')
results = list(
self.module.read_grid_data(
None,
"time:0:1,lat:0:10,lon:0:10;time:0:1,lat:10:20,lon:0:10;file://%s"
% test_file))
self.assertEquals(2, len(results))
results = [
nexusproto.NexusTile.FromString(nexus_tile_data)
for nexus_tile_data in results
]
for nexus_tile in results:
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('grid_tile'))
tile = nexus_tile.tile.grid_tile
self.assertEquals(10, len(from_shaped_array(tile.latitude)))
self.assertEquals(10, len(from_shaped_array(tile.longitude)))
the_data = np.ma.masked_invalid(
from_shaped_array(tile.variable_data))
self.assertEquals((1, 10, 10), the_data.shape)
self.assertEquals(0, np.ma.count(the_data))
self.assertTrue(tile.HasField('time'))
def process(self, tile, *args, **kwargs):
"""
This method will reverse the ordering of latitude values in a tile if necessary to ensure that the latitude values are ascending.
:param self:
:param tile: The nexus_tile
:return: Tile data with altered latitude values
"""
the_tile_type = tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(tile.tile, the_tile_type)
latitudes = from_shaped_array(the_tile_data.latitude)
data = from_shaped_array(the_tile_data.variable_data)
# Only reverse latitude ordering if current ordering is descending.
if len(latitudes) > 1:
delta = latitudes[1] - latitudes[0]
if delta < 0:
latitudes = np.flip(latitudes)
data = np.flip(data, axis=0)
the_tile_data.latitude.CopyFrom(to_shaped_array(latitudes))
the_tile_data.variable_data.CopyFrom(to_shaped_array(data))
return tile
def test_process(self):
processor = ForceAscendingLatitude()
tile = self.read_tile()
tile_type = tile.tile.WhichOneof("tile_type")
tile_data = getattr(tile.tile, tile_type)
latitudes = from_shaped_array(tile_data.latitude)
variable_data = from_shaped_array(tile_data.variable_data)
print(latitudes)
print(variable_data)
flipped_tile = processor.process(tile)
the_flipped_tile_type = flipped_tile.tile.WhichOneof("tile_type")
the_flipped_tile_data = getattr(flipped_tile.tile,
the_flipped_tile_type)
flipped_latitudes = from_shaped_array(the_flipped_tile_data.latitude)
flipped_data = from_shaped_array(the_flipped_tile_data.variable_data)
print(flipped_latitudes[1])
np.testing.assert_almost_equal(flipped_latitudes[1],
38.72608,
decimal=5,
err_msg='',
verbose=True)
print(flipped_data[1, 1])
np.testing.assert_almost_equal(flipped_data[1, 1],
0.3116,
decimal=4,
err_msg='',
verbose=True)
def test_subtraction_longitudes_greater_than_180(self):
test_file = path.join(path.dirname(__file__), 'dumped_nexustiles',
'ascat_longitude_more_than_180.bin')
with open(test_file, 'rb') as f:
nexustile_str = f.read()
nexus_tile_before = nexusproto.NexusTile.FromString(nexustile_str)
longitudes_before = from_shaped_array(
nexus_tile_before.tile.swath_tile.longitude)
subtract = sdap.processors.Subtract180Longitude()
results = list(subtract.process(nexustile_str))
self.assertEqual(1, len(results))
nexus_tile_after = results[0]
longitudes_after = from_shaped_array(
nexus_tile_after.tile.swath_tile.longitude)
self.assertTrue(
np.all(np.not_equal(longitudes_before, longitudes_after)))
self.assertTrue(np.all(longitudes_after[longitudes_after < 0]))
self.assertAlmostEqual(-96.61, longitudes_after[0][26], places=2)
def test_read_not_empty_mur(self):
test_file = path.join(path.dirname(__file__), 'datafiles',
'not_empty_mur.nc4')
results = list(
self.module.read_grid_data(
None,
"time:0:1,lat:0:10,lon:0:10;time:0:1,lat:10:20,lon:0:10;file://%s"
% test_file))
self.assertEquals(2, len(results))
results = [
nexusproto.NexusTile.FromString(nexus_tile_data)
for nexus_tile_data in results
]
tile1_data = np.ma.masked_invalid(
from_shaped_array(results[0].tile.grid_tile.variable_data))
self.assertEquals((1, 10, 10), tile1_data.shape)
self.assertEquals(100, np.ma.count(tile1_data))
tile2_data = np.ma.masked_invalid(
from_shaped_array(results[1].tile.grid_tile.variable_data))
self.assertEquals((1, 10, 10), tile2_data.shape)
self.assertEquals(100, np.ma.count(tile2_data))
self.assertFalse(np.allclose(tile1_data, tile2_data, equal_nan=True),
"Both tiles contain identical data")
def process_nexus_tile(self, nexus_tile):
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
latitudes = numpy.ma.masked_invalid(from_shaped_array(the_tile_data.latitude))
longitudes = numpy.ma.masked_invalid(from_shaped_array(the_tile_data.longitude))
data = from_shaped_array(the_tile_data.variable_data)
if nexus_tile.HasField("summary"):
tilesummary = nexus_tile.summary
else:
tilesummary = nexusproto.TileSummary()
tilesummary.bbox.lat_min = numpy.nanmin(latitudes).item()
tilesummary.bbox.lat_max = numpy.nanmax(latitudes).item()
tilesummary.bbox.lon_min = numpy.nanmin(longitudes).item()
tilesummary.bbox.lon_max = numpy.nanmax(longitudes).item()
tilesummary.stats.min = numpy.nanmin(data).item()
tilesummary.stats.max = numpy.nanmax(data).item()
# In order to accurately calculate the average we need to weight the data based on the cosine of its latitude
# This is handled slightly differently for swath vs. grid data
if the_tile_type == 'swath_tile':
# For Swath tiles, len(data) == len(latitudes) == len(longitudes). So we can simply weight each element in the
# data array
tilesummary.stats.mean = numpy.ma.average(numpy.ma.masked_invalid(data),
weights=numpy.cos(numpy.radians(latitudes))).item()
elif the_tile_type == 'grid_tile':
# Grid tiles need to repeat the weight for every longitude
# TODO This assumes data axis' are ordered as latitude x longitude
tilesummary.stats.mean = numpy.ma.average(numpy.ma.masked_invalid(data).flatten(),
weights=numpy.cos(
numpy.radians(
numpy.repeat(latitudes, len(longitudes))))).item()
else:
# Default to simple average with no weighting
tilesummary.stats.mean = numpy.nanmean(data).item()
tilesummary.stats.count = data.size - numpy.count_nonzero(numpy.isnan(data))
try:
min_time, max_time = find_time_min_max(the_tile_data)
tilesummary.stats.min_time = min_time
tilesummary.stats.max_time = max_time
except NoTimeException:
pass
try:
tilesummary.data_var_name = self.stored_var_name
except TypeError:
pass
nexus_tile.summary.CopyFrom(tilesummary)
yield nexus_tile
def get_lat_lon_time_data_meta(self):
if self._get_nexus_tile().HasField('grid_tile'):
grid_tile = self._get_nexus_tile().grid_tile
grid_tile_data = np.ma.masked_invalid(
from_shaped_array(grid_tile.variable_data))
latitude_data = np.ma.masked_invalid(
from_shaped_array(grid_tile.latitude))
longitude_data = np.ma.masked_invalid(
from_shaped_array(grid_tile.longitude))
if len(grid_tile_data.shape) == 2:
grid_tile_data = grid_tile_data[np.newaxis, :]
# Extract the meta data
meta_data = {}
for meta_data_obj in grid_tile.meta_data:
name = meta_data_obj.name
meta_array = np.ma.masked_invalid(
from_shaped_array(meta_data_obj.meta_data))
if len(meta_array.shape) == 2:
meta_array = meta_array[np.newaxis, :]
meta_data[name] = meta_array
return latitude_data, longitude_data, np.array(
[grid_tile.time]), grid_tile_data, meta_data
elif self._get_nexus_tile().HasField('swath_tile'):
swath_tile = self._get_nexus_tile().swath_tile
latitude_data = np.ma.masked_invalid(
from_shaped_array(swath_tile.latitude)).reshape(-1)
longitude_data = np.ma.masked_invalid(
from_shaped_array(swath_tile.longitude)).reshape(-1)
time_data = np.ma.masked_invalid(from_shaped_array(
swath_tile.time)).reshape(-1)
# Simplify the tile if the time dimension is the same value repeated
if np.all(time_data == np.min(time_data)):
time_data = np.array([np.min(time_data)])
swath_tile_data = np.ma.masked_invalid(
from_shaped_array(swath_tile.variable_data))
tile_data = self._to_standard_index(
swath_tile_data,
(len(time_data), len(latitude_data), len(longitude_data)))
# Extract the meta data
meta_data = {}
for meta_data_obj in swath_tile.meta_data:
name = meta_data_obj.name
actual_meta_array = np.ma.masked_invalid(
from_shaped_array(meta_data_obj.meta_data))
reshaped_meta_array = self._to_standard_index(
actual_meta_array, tile_data.shape)
meta_data[name] = reshaped_meta_array
return latitude_data, longitude_data, time_data, tile_data, meta_data
else:
raise NotImplementedError("Only supports grid_tile and swath_tile")
def process_nexus_tile(self, nexus_tile):
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
wind_speed = from_shaped_array(the_tile_data.variable_data)
wind_dir = from_shaped_array(
next(meta for meta in the_tile_data.meta_data
if meta.name == 'wind_dir').meta_data)
assert wind_speed.shape == wind_dir.shape
wind_u_component = numpy.ma.empty(wind_speed.shape, dtype=float)
wind_v_component = numpy.ma.empty(wind_speed.shape, dtype=float)
wind_speed_iter = numpy.nditer(wind_speed, flags=['multi_index'])
while not wind_speed_iter.finished:
speed = wind_speed_iter[0]
current_index = wind_speed_iter.multi_index
direction = wind_dir[current_index]
# Convert degrees to radians
direction = radians(direction)
# Calculate component values
wind_u_component[current_index] = calculate_u_component_value(
direction, speed)
wind_v_component[current_index] = calculate_v_component_value(
direction, speed)
wind_speed_iter.iternext()
# Stick the original data into the meta data
wind_speed_meta = the_tile_data.meta_data.add()
wind_speed_meta.name = 'wind_speed'
wind_speed_meta.meta_data.CopyFrom(to_shaped_array(wind_speed))
# The u_or_v variable specifies which component variable is the 'data variable' for this tile
# Replace data with the appropriate component value and put the other component in metadata
if self.u_or_v == U_OR_V_ENUM.U:
the_tile_data.variable_data.CopyFrom(
to_shaped_array(wind_u_component))
wind_component_meta = the_tile_data.meta_data.add()
wind_component_meta.name = 'wind_v'
wind_component_meta.meta_data.CopyFrom(
to_shaped_array(wind_v_component))
elif self.u_or_v == U_OR_V_ENUM.V:
the_tile_data.variable_data.CopyFrom(
to_shaped_array(wind_v_component))
wind_component_meta = the_tile_data.meta_data.add()
wind_component_meta.name = 'wind_u'
wind_component_meta.meta_data.CopyFrom(
to_shaped_array(wind_u_component))
yield nexus_tile
def process(self, tile, dataset, *args, **kwargs):
tile_type = tile.tile.WhichOneof("tile_type")
tile_data = getattr(tile.tile, tile_type)
latitudes = numpy.ma.masked_invalid(
from_shaped_array(tile_data.latitude))
longitudes = numpy.ma.masked_invalid(
from_shaped_array(tile_data.longitude))
data = from_shaped_array(tile_data.variable_data)
tile_summary = tile.summary if tile.HasField(
"summary") else nexusproto.TileSummary()
tile_summary.dataset_name = self._dataset_name
tile_summary.bbox.lat_min = numpy.nanmin(latitudes).item()
tile_summary.bbox.lat_max = numpy.nanmax(latitudes).item()
tile_summary.bbox.lon_min = numpy.nanmin(longitudes).item()
tile_summary.bbox.lon_max = numpy.nanmax(longitudes).item()
tile_summary.stats.min = numpy.nanmin(data).item()
tile_summary.stats.max = numpy.nanmax(data).item()
tile_summary.stats.count = data.size - numpy.count_nonzero(
numpy.isnan(data))
# In order to accurately calculate the average we need to weight the data based on the cosine of its latitude
# This is handled slightly differently for swath vs. grid data
if tile_type == 'swath_tile':
# For Swath tiles, len(data) == len(latitudes) == len(longitudes).
# So we can simply weight each element in the data array
tile_summary.stats.mean = type(self).calculate_mean_for_swath_tile(
data, latitudes)
elif tile_type == 'grid_tile':
# Grid tiles need to repeat the weight for every longitude
# TODO This assumes data axis' are ordered as latitude x longitude
tile_summary.stats.mean = type(self).calculate_mean_for_grid_tile(
data, latitudes, longitudes)
else:
# Default to simple average with no weighting
tile_summary.stats.mean = numpy.nanmean(data).item()
try:
min_time, max_time = find_time_min_max(tile_data)
tile_summary.stats.min_time = min_time
tile_summary.stats.max_time = max_time
except NoTimeException:
pass
standard_name = dataset.variables[
tile_summary.data_var_name].attrs.get('standard_name')
if standard_name:
tile_summary.standard_name = standard_name
tile.summary.CopyFrom(tile_summary)
return tile
def transform(self, tile_data):
nexus_tile = nexusproto.NexusTile.FromString(tile_data)
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
# Either wind_u or wind_v are in meta. Whichever is not in meta is in variable_data
try:
wind_v = next(meta for meta in the_tile_data.meta_data
if meta.name == wind_v_var_name).meta_data
wind_u = the_tile_data.variable_data
except StopIteration:
try:
wind_u = next(meta for meta in the_tile_data.meta_data
if meta.name == wind_u_var_name).meta_data
wind_v = the_tile_data.variable_data
except StopIteration:
if hasattr(nexus_tile, "summary"):
raise RuntimeError(
"Neither wind_u nor wind_v were found in the meta data for granule %s slice %s."
" Cannot compute wind speed or direction." %
(getattr(nexus_tile.summary, "granule", "unknown"),
getattr(nexus_tile.summary, "section_spec", "unknown")))
else:
raise RuntimeError(
"Neither wind_u nor wind_v were found in the meta data. Cannot compute wind speed or direction."
)
wind_u = from_shaped_array(wind_u)
wind_v = from_shaped_array(wind_v)
assert wind_u.shape == wind_v.shape
# Do calculation
wind_speed_data, wind_dir_data = calculate_speed_direction(wind_u, wind_v)
# Add wind_speed to meta data
wind_speed_meta = the_tile_data.meta_data.add()
wind_speed_meta.name = 'wind_speed'
wind_speed_meta.meta_data.CopyFrom(to_shaped_array(wind_speed_data))
# Add wind_dir to meta data
wind_dir_meta = the_tile_data.meta_data.add()
wind_dir_meta.name = 'wind_dir'
wind_dir_meta.meta_data.CopyFrom(to_shaped_array(wind_dir_data))
yield nexus_tile.SerializeToString()
def test_read_not_empty_ascatb_meta(self):
# with open('./ascat_longitude_more_than_180.bin', 'w') as f:
# results = list(self.module.read_swath_data(None,
# "NUMROWS:0:1,NUMCELLS:0:82;NUMROWS:1:2,NUMCELLS:0:82;file:///Users/greguska/Downloads/ascat_longitude_more_than_180.nc4"))
# f.write(results[0])
test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_ascatb.nc4')
swath_reader = sdap.processors.SwathReadingProcessor('wind_speed', 'lat', 'lon', time='time', meta='wind_dir')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "NUMROWS:0:1,NUMCELLS:0:82"
input_tile.summary.CopyFrom(tile_summary)
results = list(swath_reader.process(input_tile))
self.assertEqual(1, len(results))
for nexus_tile in results:
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
self.assertLess(0, len(nexus_tile.tile.swath_tile.meta_data))
self.assertEqual(1, len(results[0].tile.swath_tile.meta_data))
tile1_meta_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.meta_data[0].meta_data))
self.assertEqual((1, 82), tile1_meta_data.shape)
self.assertEqual(82, np.ma.count(tile1_meta_data))
def test_read_not_empty_ascatb_meta(self):
environ['META'] = 'wind_dir'
reload(self.module)
# with open('./ascat_longitude_more_than_180.bin', 'w') as f:
# results = list(self.module.read_swath_data(None,
# "NUMROWS:0:1,NUMCELLS:0:82;NUMROWS:1:2,NUMCELLS:0:82;file:///Users/greguska/Downloads/ascat_longitude_more_than_180.nc4"))
# f.write(results[0])
test_file = path.join(path.dirname(__file__), 'datafiles',
'not_empty_ascatb.nc4')
results = list(
self.module.read_swath_data(
None,
"NUMROWS:0:1,NUMCELLS:0:82;NUMROWS:1:2,NUMCELLS:0:82;file://%s"
% test_file))
self.assertEquals(2, len(results))
results = [
nexusproto.NexusTile.FromString(nexus_tile_data)
for nexus_tile_data in results
]
for nexus_tile in results:
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
self.assertLess(0, len(nexus_tile.tile.swath_tile.meta_data))
self.assertEquals(1, len(results[0].tile.swath_tile.meta_data))
tile1_meta_data = np.ma.masked_invalid(
from_shaped_array(
results[0].tile.swath_tile.meta_data[0].meta_data))
self.assertEquals((1, 82), tile1_meta_data.shape)
self.assertEquals(82, np.ma.count(tile1_meta_data))
self.assertEquals(1, len(results[1].tile.swath_tile.meta_data))
tile2_meta_data = np.ma.masked_invalid(
from_shaped_array(
results[1].tile.swath_tile.meta_data[0].meta_data))
self.assertEquals((1, 82), tile2_meta_data.shape)
self.assertEquals(82, np.ma.count(tile2_meta_data))
self.assertFalse(
np.allclose(tile1_meta_data, tile2_meta_data, equal_nan=True),
"Both tiles' meta contain identical data")
def get_raw_data_array(self):
nexus_tile = self._get_nexus_tile()
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
return from_shaped_array(the_tile_data.variable_data)
def test_u_conversion(self):
test_file = path.join(path.dirname(__file__), 'dumped_nexustiles',
'ascatb_nonempty_nexustile.bin')
with open(test_file, 'rb') as f:
nexustile_str = f.read()
converter = sdap.processors.WindDirSpeedToUV('V')
results = list(converter.process(nexustile_str))
self.assertEqual(1, len(results))
nexus_tile = results[0]
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
# Check data
tile_data = np.ma.masked_invalid(
from_shaped_array(nexus_tile.tile.swath_tile.variable_data))
self.assertEqual(82, np.ma.count(tile_data))
# Check meta data
meta_list = nexus_tile.tile.swath_tile.meta_data
self.assertEqual(3, len(meta_list))
wind_dir = next(meta_obj for meta_obj in meta_list
if meta_obj.name == 'wind_dir')
self.assertEqual(
tile_data.shape,
np.ma.masked_invalid(from_shaped_array(wind_dir.meta_data)).shape)
self.assertIsNotNone(wind_dir)
wind_speed = next(meta_obj for meta_obj in meta_list
if meta_obj.name == 'wind_speed')
self.assertIsNotNone(wind_speed)
self.assertEqual(
tile_data.shape,
np.ma.masked_invalid(from_shaped_array(
wind_speed.meta_data)).shape)
wind_u = next(meta_obj for meta_obj in meta_list
if meta_obj.name == 'wind_u')
self.assertIsNotNone(wind_u)
self.assertEqual(
tile_data.shape,
np.ma.masked_invalid(from_shaped_array(wind_u.meta_data)).shape)
def test_read_not_empty_wswm(self):
test_file = path.join(path.dirname(__file__), 'datafiles',
'not_empty_wswm.nc')
wswm_reader = sdap.processors.TimeSeriesReadingProcessor(
'Qout', 'lat', 'lon', 'time')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "time:0:5832,rivid:0:1"
input_tile.summary.CopyFrom(tile_summary)
results = list(wswm_reader.process(input_tile))
self.assertEqual(1, len(results))
for nexus_tile in results:
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('time_series_tile'))
tile = nexus_tile.tile.time_series_tile
self.assertEqual(1, from_shaped_array(tile.latitude).size)
self.assertEqual(1, from_shaped_array(tile.longitude).size)
self.assertEqual((5832, 1),
from_shaped_array(tile.variable_data).shape)
tile1_data = np.ma.masked_invalid(
from_shaped_array(results[0].tile.time_series_tile.variable_data))
self.assertEqual(5832, np.ma.count(tile1_data))
self.assertAlmostEqual(
45.837,
np.ma.min(
np.ma.masked_invalid(
from_shaped_array(
results[0].tile.time_series_tile.latitude))),
places=3)
self.assertAlmostEqual(
-122.789,
np.ma.max(
np.ma.masked_invalid(
from_shaped_array(
results[0].tile.time_series_tile.longitude))),
places=3)
tile1_times = from_shaped_array(results[0].tile.time_series_tile.time)
self.assertEqual(852098400, tile1_times[0])
self.assertEqual(915073200, tile1_times[-1])
self.assertAlmostEqual(
1.473,
np.ma.masked_invalid(
from_shaped_array(
results[0].tile.time_series_tile.variable_data))[0, 0],
places=3)
def test_u_conversion(self):
test_file = path.join(path.dirname(__file__), 'dumped_nexustiles',
'ascatb_nonempty_nexustile.bin')
with open(test_file, 'r') as f:
nexustile_str = f.read()
results = list(self.module.transform(None, nexustile_str))
self.assertEquals(1, len(results))
nexus_tile = nexusproto.NexusTile.FromString(results[0])
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
# Check data
tile_data = np.ma.masked_invalid(
from_shaped_array(nexus_tile.tile.swath_tile.variable_data))
self.assertEquals(82, np.ma.count(tile_data))
# Check meta data
meta_list = nexus_tile.tile.swath_tile.meta_data
self.assertEquals(3, len(meta_list))
wind_dir = next(meta_obj for meta_obj in meta_list
if meta_obj.name == 'wind_dir')
self.assertEquals(
tile_data.shape,
np.ma.masked_invalid(from_shaped_array(wind_dir.meta_data)).shape)
self.assertIsNotNone(wind_dir)
wind_speed = next(meta_obj for meta_obj in meta_list
if meta_obj.name == 'wind_speed')
self.assertIsNotNone(wind_speed)
self.assertEquals(
tile_data.shape,
np.ma.masked_invalid(from_shaped_array(
wind_speed.meta_data)).shape)
wind_u = next(meta_obj for meta_obj in meta_list
if meta_obj.name == 'wind_u')
self.assertIsNotNone(wind_u)
self.assertEquals(
tile_data.shape,
np.ma.masked_invalid(from_shaped_array(wind_u.meta_data)).shape)
def test_read_not_empty_smap(self):
test_file = path.join(path.dirname(__file__), 'datafiles',
'not_empty_smap.h5')
results = list(
self.module.read_swath_data(
None,
"phony_dim_0:0:76,phony_dim_1:0:1;phony_dim_0:0:76,phony_dim_1:1:2;file://%s"
% test_file))
self.assertEquals(2, len(results))
# with open('./smap_nonempty_nexustile.bin', 'w') as f:
# f.write(results[0])
results = [
nexusproto.NexusTile.FromString(nexus_tile_data)
for nexus_tile_data in results
]
for nexus_tile in results:
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
self.assertEquals(0, len(nexus_tile.tile.swath_tile.meta_data))
tile = nexus_tile.tile.swath_tile
self.assertEquals(76, from_shaped_array(tile.latitude).size)
self.assertEquals(76, from_shaped_array(tile.longitude).size)
tile1_data = np.ma.masked_invalid(
from_shaped_array(results[0].tile.swath_tile.variable_data))
self.assertEquals((76, 1), tile1_data.shape)
self.assertEquals(43, np.ma.count(tile1_data))
self.assertAlmostEquals(
-50.056,
np.ma.min(
np.ma.masked_invalid(
from_shaped_array(results[0].tile.swath_tile.latitude))),
places=3)
self.assertAlmostEquals(
-47.949,
np.ma.max(
np.ma.masked_invalid(
from_shaped_array(results[0].tile.swath_tile.latitude))),
places=3)
tile2_data = np.ma.masked_invalid(
from_shaped_array(results[1].tile.swath_tile.variable_data))
self.assertEquals((76, 1), tile2_data.shape)
self.assertEquals(43, np.ma.count(tile2_data))
self.assertFalse(np.allclose(tile1_data, tile2_data, equal_nan=True),
"Both tiles contain identical data")
self.assertEquals(
1427820162,
np.ma.masked_invalid(
from_shaped_array(results[0].tile.swath_tile.time))[0])
def find_time_min_max(tile_data):
if tile_data.time:
if isinstance(tile_data.time, nexusproto.ShapedArray):
time_data = from_shaped_array(tile_data.time)
return int(numpy.nanmin(time_data).item()), int(
numpy.nanmax(time_data).item())
elif isinstance(tile_data.time, int):
return tile_data.time, tile_data.time
raise NoTimeException
def process_nexus_tile(self, nexus_tile):
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
var_data = from_shaped_array(the_tile_data.variable_data) - 273.15
the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data))
yield nexus_tile
def test_subtraction_longitudes_less_than_180(self):
test_file = path.join(path.dirname(__file__), 'dumped_nexustiles', 'ascatb_nonempty_nexustile.bin')
with open(test_file, 'rb') as f:
nexustile_str = f.read()
nexus_tile_before = nexusproto.NexusTile.FromString(nexustile_str)
longitudes_before = from_shaped_array(nexus_tile_before.tile.swath_tile.longitude)
subtract = processors.Subtract180Longitude()
results = list(subtract.process(nexustile_str))
self.assertEqual(1, len(results))
nexus_tile_after = results[0]
longitudes_after = from_shaped_array(nexus_tile_after.tile.swath_tile.longitude)
self.assertTrue(np.all(np.equal(longitudes_before, longitudes_after)))
def test_speed_dir_computation(self):
test_file = path.join(path.dirname(__file__), 'dumped_nexustiles',
'ccmp_nonempty_nexustile.bin')
with open(test_file, 'rb') as f:
nexustile_str = f.read()
results = list(self.module.process(nexustile_str))
self.assertEqual(1, len(results))
nexus_tile = results[0]
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('grid_tile'))
# Check data
tile_data = np.ma.masked_invalid(
from_shaped_array(nexus_tile.tile.grid_tile.variable_data))
self.assertEqual(3306, np.ma.count(tile_data))
# Check meta data
meta_list = nexus_tile.tile.grid_tile.meta_data
self.assertEqual(3, len(meta_list))
wind_dir = next(meta_obj for meta_obj in meta_list
if meta_obj.name == 'wind_dir')
self.assertEqual(
tile_data.shape,
np.ma.masked_invalid(from_shaped_array(wind_dir.meta_data)).shape)
self.assertIsNotNone(wind_dir)
wind_speed = next(meta_obj for meta_obj in meta_list
if meta_obj.name == 'wind_speed')
self.assertIsNotNone(wind_speed)
self.assertEqual(
tile_data.shape,
np.ma.masked_invalid(from_shaped_array(
wind_speed.meta_data)).shape)
wind_v = next(meta_obj for meta_obj in meta_list
if meta_obj.name == 'vwnd')
self.assertIsNotNone(wind_v)
self.assertEqual(
tile_data.shape,
np.ma.masked_invalid(from_shaped_array(wind_v.meta_data)).shape)
def find_time_min_max(tile_data):
# Only try to grab min/max time if it exists as a ShapedArray
if tile_data.HasField("time") and isinstance(tile_data.time, nexusproto.ShapedArray):
time_data = from_shaped_array(tile_data.time)
min_time = long(numpy.nanmin(time_data).item())
max_time = long(numpy.nanmax(time_data).item())
return min_time, max_time
elif tile_data.HasField("time") and isinstance(tile_data.time, (int, long)):
return tile_data.time, tile_data.time
raise NoTimeException
def transform(self, tile_data):
nexus_tile = nexusproto.NexusTile.FromString(tile_data)
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
longitudes = from_shaped_array(the_tile_data.longitude) - 180
the_tile_data.longitude.CopyFrom(to_shaped_array(longitudes))
yield nexus_tile.SerializeToString()
def transform(self, tile_data):
nexus_tile = nexusproto.NexusTile.FromString(tile_data)
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
var_data = from_shaped_array(the_tile_data.variable_data) - 273.15
the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data))
yield nexus_tile.SerializeToString()
def summarize_nexustile(self, tiledata):
nexus_tile = parse_input(tiledata)
the_tile_type = nexus_tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(nexus_tile.tile, the_tile_type)
latitudes = numpy.ma.masked_invalid(from_shaped_array(the_tile_data.latitude))
longitudes = numpy.ma.masked_invalid(from_shaped_array(the_tile_data.longitude))
data = from_shaped_array(the_tile_data.variable_data)
if nexus_tile.HasField("summary"):
tilesummary = nexus_tile.summary
else:
tilesummary = nexusproto.TileSummary()
tilesummary.bbox.lat_min = numpy.nanmin(latitudes).item()
tilesummary.bbox.lat_max = numpy.nanmax(latitudes).item()
tilesummary.bbox.lon_min = numpy.nanmin(longitudes).item()
tilesummary.bbox.lon_max = numpy.nanmax(longitudes).item()
tilesummary.stats.min = numpy.nanmin(data).item()
tilesummary.stats.max = numpy.nanmax(data).item()
tilesummary.stats.mean = numpy.nanmean(data).item()
tilesummary.stats.count = data.size - numpy.count_nonzero(numpy.isnan(data))
try:
min_time, max_time = find_time_min_max(the_tile_data)
tilesummary.stats.min_time = min_time
tilesummary.stats.max_time = max_time
except NoTimeException:
pass
try:
tilesummary.data_var_name = var_name
except NameError:
pass
nexus_tile.summary.CopyFrom(tilesummary)
yield nexus_tile.SerializeToString()
def process(self, tile, *args, **kwargs):
tile_type = tile.tile.WhichOneof("tile_type")
tile_data = getattr(tile.tile, tile_type)
data = from_shaped_array(tile_data.variable_data)
# Only supply data if there is actual values in the tile
if data.size - numpy.count_nonzero(numpy.isnan(data)) > 0:
return tile
else:
logger.warning(
"Discarding tile from {} because it is empty".format(
tile.summary.granule))
return None
def test_delete_time_axis(self):
test_file = path.join(path.dirname(__file__), 'dumped_nexustiles',
'avhrr_nonempty_nexustile.bin')
with open(test_file, 'rb') as f:
nexustile_str = f.read()
nexus_tile_before = nexusproto.NexusTile.FromString(nexustile_str)
sst_before = from_shaped_array(
nexus_tile_before.tile.grid_tile.variable_data)
self.assertEqual((1, 10, 10), sst_before.shape)
results = list(self.module.process(nexustile_str))
nexus_tile_after = results[0]
sst_after = from_shaped_array(
nexus_tile_after.tile.grid_tile.variable_data)
self.assertEqual((10, 10), sst_after.shape)