def test_read_empty_mur(self):
test_file = path.join(path.dirname(__file__), 'datafiles', 'empty_mur.nc4')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "time:0:1,lat:0:10,lon:0:10"
input_tile.summary.CopyFrom(tile_summary)
results = list(self.module.process(input_tile))
self.assertEqual(1, len(results))
for nexus_tile in results:
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('grid_tile'))
tile = nexus_tile.tile.grid_tile
self.assertEqual(10, len(from_shaped_array(tile.latitude)))
self.assertEqual(10, len(from_shaped_array(tile.longitude)))
the_data = np.ma.masked_invalid(from_shaped_array(tile.variable_data))
self.assertEqual((1, 10, 10), the_data.shape)
self.assertEqual(0, np.ma.count(the_data))
self.assertTrue(tile.time)
def test_generate_tile_with_dims_out_of_order(self):
reading_processor = EccoReadingProcessor(variable='OBP',
latitude='YC',
longitude='XC',
time='time',
tile='tile')
granule_path = path.join(path.dirname(__file__),
'../granules/OBP_native_grid.nc')
input_tile = nexusproto.NexusTile()
dimensions_to_slices = {
'j': slice(0, 15),
'tile': slice(10, 11),
'i': slice(0, 7),
'time': slice(0, 1)
}
with xr.open_dataset(granule_path, decode_cf=True) as ds:
output_tile = reading_processor._generate_tile(
ds, dimensions_to_slices, input_tile)
self.assertEqual(output_tile.tile.ecco_tile.tile, 10)
self.assertEqual(output_tile.tile.ecco_tile.time, 695563200)
self.assertEqual(output_tile.tile.ecco_tile.variable_data.shape,
[15, 7])
self.assertEqual(output_tile.tile.ecco_tile.latitude.shape,
[15, 7])
self.assertEqual(output_tile.tile.ecco_tile.longitude.shape,
[15, 7])
def test_run_chain_read_filter_all(self):
processor_list = [{
'name': 'GridReadingProcessor',
'config': {
'latitude': 'lat',
'longitude': 'lon',
'time': 'time',
'variable_to_read': 'analysed_sst'
}
}, {
'name': 'EmptyTileFilter',
'config': {}
}]
processorchain = ProcessorChain(processor_list)
test_file = path.join(path.dirname(__file__), 'datafiles',
'empty_mur.nc4')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "time:0:1,lat:0:10,lon:0:10"
input_tile.summary.CopyFrom(tile_summary)
gen = processorchain.process(input_tile)
for message in gen:
self.fail("Should not produce any messages. Message: %s" % message)
def test_read_not_empty_ascatb_meta(self):
# with open('./ascat_longitude_more_than_180.bin', 'w') as f:
# results = list(self.module.read_swath_data(None,
# "NUMROWS:0:1,NUMCELLS:0:82;NUMROWS:1:2,NUMCELLS:0:82;file:///Users/greguska/Downloads/ascat_longitude_more_than_180.nc4"))
# f.write(results[0])
test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_ascatb.nc4')
swath_reader = sdap.processors.SwathReadingProcessor('wind_speed', 'lat', 'lon', time='time', meta='wind_dir')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "NUMROWS:0:1,NUMCELLS:0:82"
input_tile.summary.CopyFrom(tile_summary)
results = list(swath_reader.process(input_tile))
self.assertEqual(1, len(results))
for nexus_tile in results:
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
self.assertLess(0, len(nexus_tile.tile.swath_tile.meta_data))
self.assertEqual(1, len(results[0].tile.swath_tile.meta_data))
tile1_meta_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.meta_data[0].meta_data))
self.assertEqual((1, 82), tile1_meta_data.shape)
self.assertEqual(82, np.ma.count(tile1_meta_data))
def test_run_chain_read_filter_kelvin_summarize(self):
processor_list = [{
'name': 'GridReadingProcessor',
'config': {
'latitude': 'lat',
'longitude': 'lon',
'time': 'time',
'variable_to_read': 'analysed_sst'
}
}, {
'name': 'EmptyTileFilter',
'config': {}
}, {
'name': 'KelvinToCelsius',
'config': {}
}, {
'name': 'TileSummarizingProcessor',
'config': {}
}]
processorchain = ProcessorChain(processor_list)
test_file = path.join(path.dirname(__file__), 'datafiles',
'not_empty_mur.nc4')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "time:0:1,lat:0:10,lon:0:10"
input_tile.summary.CopyFrom(tile_summary)
results = list(processorchain.process(input_tile))
self.assertEqual(1, len(results))
def test_read_not_empty_ascatb(self):
reading_processor = SwathReadingProcessor(variable='wind_speed',
latitude='lat',
longitude='lon',
time='time')
granule_path = path.join(path.dirname(__file__),
'../granules/not_empty_ascatb.nc4')
input_tile = nexusproto.NexusTile()
input_tile.summary.granule = granule_path
dimensions_to_slices = {
'NUMROWS': slice(0, 1),
'NUMCELLS': slice(0, 82)
}
with xr.open_dataset(granule_path, decode_cf=True) as ds:
output_tile = reading_processor._generate_tile(
ds, dimensions_to_slices, input_tile)
self.assertEqual(granule_path, output_tile.summary.granule,
granule_path)
self.assertEqual([1, 82], output_tile.tile.swath_tile.time.shape)
self.assertEqual([1, 82],
output_tile.tile.swath_tile.variable_data.shape)
self.assertEqual([1, 82],
output_tile.tile.swath_tile.latitude.shape)
self.assertEqual([1, 82],
output_tile.tile.swath_tile.longitude.shape)
def test_read_not_empty_smap(self):
reading_processor = SwathReadingProcessor(variable='smap_sss',
latitude='lat',
longitude='lon',
time='row_time')
granule_path = path.join(path.dirname(__file__),
'../granules/not_empty_smap.h5')
input_tile = nexusproto.NexusTile()
input_tile.summary.granule = granule_path
dimensions_to_slices = {
'phony_dim_0': slice(0, 38),
'phony_dim_1': slice(0, 1)
}
with xr.open_dataset(granule_path, decode_cf=True) as ds:
output_tile = reading_processor._generate_tile(
ds, dimensions_to_slices, input_tile)
self.assertEqual(granule_path, output_tile.summary.granule)
self.assertEqual([1], output_tile.tile.swath_tile.time.shape)
self.assertEqual([38, 1],
output_tile.tile.swath_tile.variable_data.shape)
self.assertEqual([38, 1],
output_tile.tile.swath_tile.latitude.shape)
self.assertEqual([38, 1],
output_tile.tile.swath_tile.longitude.shape)
def run_processor_chain():
try:
parameters = request.get_json()
except Exception as e:
raise BadRequest("Invalid JSON data") from e
try:
processor_list = parameters['processor_list']
except (KeyError, TypeError):
raise BadRequest(description="processor_list is required.")
try:
chain = ProcessorChain(processor_list)
except ProcessorNotFound as e:
raise BadRequest("Unknown processor requested: %s" %
e.missing_processor) from e
except MissingProcessorArguments as e:
raise BadRequest("%s missing required configuration options: %s" %
(e.processor, e.missing_processor_args)) from e
try:
input_data = json_format.Parse(parameters['input_data'],
nexusproto.NexusTile())
except ParseError as e:
raise BadRequest(
"input_data must be a NexusTile protobuf serialized as a string"
) from e
result = next(chain.process(input_data), None)
if isinstance(result, nexusproto.NexusTile):
result = result.SerializeToString()
return Response(result, mimetype='application/octet-stream')
def test_read_not_empty_ascatb(self):
test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_ascatb.nc4')
swath_reader = sdap.processors.SwathReadingProcessor('wind_speed', 'lat', 'lon', time='time')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "NUMROWS:0:1,NUMCELLS:0:82"
input_tile.summary.CopyFrom(tile_summary)
results = list(swath_reader.process(input_tile))
self.assertEqual(1, len(results))
for nexus_tile in results:
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
self.assertEqual(0, len(nexus_tile.tile.swath_tile.meta_data))
tile = nexus_tile.tile.swath_tile
self.assertEqual(82, from_shaped_array(tile.latitude).size)
self.assertEqual(82, from_shaped_array(tile.longitude).size)
tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.swath_tile.variable_data))
self.assertEqual((1, 82), tile1_data.shape)
self.assertEqual(82, np.ma.count(tile1_data))
def test_standard_name_exists(self):
"""
Test that the standard_name attribute exists in a
Tile.TileSummary object after being processed with
TileSummarizingProcessor
"""
reading_processor = GridReadingProcessor(
variable='analysed_sst',
latitude='lat',
longitude='lon',
time='time',
tile='tile'
)
relative_path = '../granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc'
granule_path = path.join(path.dirname(__file__), relative_path)
tile_summary = nexusproto.TileSummary()
tile_summary.granule = granule_path
tile_summary.data_var_name = 'analysed_sst'
input_tile = nexusproto.NexusTile()
input_tile.summary.CopyFrom(tile_summary)
dims = {
'lat': slice(0, 30),
'lon': slice(0, 30),
'time': slice(0, 1),
'tile': slice(10, 11),
}
with xr.open_dataset(granule_path, decode_cf=True) as ds:
output_tile = reading_processor._generate_tile(ds, dims, input_tile)
tile_summary_processor = TileSummarizingProcessor('test')
new_tile = tile_summary_processor.process(tile=output_tile, dataset=ds)
assert new_tile.summary.standard_name == 'sea_surface_temperature'
def test_read_not_empty_smap(self):
test_file = path.join(path.dirname(__file__), 'datafiles',
'not_empty_smap.h5')
swath_reader = sdap.processors.SwathReadingProcessor(
'smap_sss',
'lat',
'lon',
time='row_time',
glblattr_day='REV_START_TIME',
glblattr_day_format='%Y-%jT%H:%M:%S.%f')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "phony_dim_0:0:76,phony_dim_1:0:1"
input_tile.summary.CopyFrom(tile_summary)
results = list(swath_reader.process(input_tile))
self.assertEqual(1, len(results))
# with open('./smap_nonempty_nexustile.bin', 'w') as f:
# f.write(results[0])
for nexus_tile in results:
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('swath_tile'))
self.assertEqual(0, len(nexus_tile.tile.swath_tile.meta_data))
tile = nexus_tile.tile.swath_tile
self.assertEqual(76, from_shaped_array(tile.latitude).size)
self.assertEqual(76, from_shaped_array(tile.longitude).size)
tile1_data = np.ma.masked_invalid(
from_shaped_array(results[0].tile.swath_tile.variable_data))
self.assertEqual((76, 1), tile1_data.shape)
self.assertEqual(43, np.ma.count(tile1_data))
self.assertAlmostEqual(
-50.056,
np.ma.min(
np.ma.masked_invalid(
from_shaped_array(results[0].tile.swath_tile.latitude))),
places=3)
self.assertAlmostEqual(
-47.949,
np.ma.max(
np.ma.masked_invalid(
from_shaped_array(results[0].tile.swath_tile.latitude))),
places=3)
self.assertEqual(
1427820162,
np.ma.masked_invalid(
from_shaped_array(results[0].tile.swath_tile.time))[0])
def test_read_not_empty_wswm(self):
test_file = path.join(path.dirname(__file__), 'datafiles',
'not_empty_wswm.nc')
wswm_reader = sdap.processors.TimeSeriesReadingProcessor(
'Qout', 'lat', 'lon', 'time')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "time:0:5832,rivid:0:1"
input_tile.summary.CopyFrom(tile_summary)
results = list(wswm_reader.process(input_tile))
self.assertEqual(1, len(results))
for nexus_tile in results:
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('time_series_tile'))
tile = nexus_tile.tile.time_series_tile
self.assertEqual(1, from_shaped_array(tile.latitude).size)
self.assertEqual(1, from_shaped_array(tile.longitude).size)
self.assertEqual((5832, 1),
from_shaped_array(tile.variable_data).shape)
tile1_data = np.ma.masked_invalid(
from_shaped_array(results[0].tile.time_series_tile.variable_data))
self.assertEqual(5832, np.ma.count(tile1_data))
self.assertAlmostEqual(
45.837,
np.ma.min(
np.ma.masked_invalid(
from_shaped_array(
results[0].tile.time_series_tile.latitude))),
places=3)
self.assertAlmostEqual(
-122.789,
np.ma.max(
np.ma.masked_invalid(
from_shaped_array(
results[0].tile.time_series_tile.longitude))),
places=3)
tile1_times = from_shaped_array(results[0].tile.time_series_tile.time)
self.assertEqual(852098400, tile1_times[0])
self.assertEqual(915073200, tile1_times[-1])
self.assertAlmostEqual(
1.473,
np.ma.masked_invalid(
from_shaped_array(
results[0].tile.time_series_tile.variable_data))[0, 0],
places=3)
def test_read_not_empty_avhrr(self):
test_file = path.join(path.dirname(__file__), 'datafiles',
'not_empty_avhrr.nc4')
avhrr_reader = sdap.processors.GridReadingProcessor('analysed_sst',
'lat',
'lon',
time='time')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "time:0:1,lat:0:10,lon:0:10"
input_tile.summary.CopyFrom(tile_summary)
results = list(avhrr_reader.process(input_tile))
self.assertEqual(1, len(results))
for nexus_tile in results:
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('grid_tile'))
tile = nexus_tile.tile.grid_tile
self.assertEqual(10, from_shaped_array(tile.latitude).size)
self.assertEqual(10, from_shaped_array(tile.longitude).size)
self.assertEqual((1, 10, 10),
from_shaped_array(tile.variable_data).shape)
tile1_data = np.ma.masked_invalid(
from_shaped_array(results[0].tile.grid_tile.variable_data))
self.assertEqual(100, np.ma.count(tile1_data))
self.assertAlmostEqual(
-39.875,
np.ma.min(
np.ma.masked_invalid(
from_shaped_array(results[0].tile.grid_tile.latitude))),
places=3)
self.assertAlmostEqual(
-37.625,
np.ma.max(
np.ma.masked_invalid(
from_shaped_array(results[0].tile.grid_tile.latitude))),
places=3)
self.assertEqual(1462060800, results[0].tile.grid_tile.time)
self.assertAlmostEqual(
289.71,
np.ma.masked_invalid(
from_shaped_array(results[0].tile.grid_tile.variable_data))[0,
0,
0],
places=3)
def process_nexus_tile(self, input_tile):
tile_specifications, file_path = parse_input(input_tile, self.temp_dir)
output_tile = nexusproto.NexusTile()
output_tile.CopyFrom(input_tile)
for tile in self.read_data(tile_specifications, file_path, output_tile):
yield tile
# If temp dir is defined, delete the temporary file
if self.temp_dir is not None:
remove(file_path)
def test_read_not_empty_ccmp(self):
test_file = path.join(path.dirname(__file__), 'datafiles',
'not_empty_ccmp.nc')
ccmp_reader = sdap.processors.GridReadingProcessor('uwnd',
'latitude',
'longitude',
time='time',
meta='vwnd')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "time:0:1,longitude:0:87,latitude:0:38"
input_tile.summary.CopyFrom(tile_summary)
results = list(ccmp_reader.process(input_tile))
self.assertEqual(1, len(results))
# with open('./ccmp_nonempty_nexustile.bin', 'w') as f:
# f.write(results[0])
for nexus_tile in results:
self.assertTrue(nexus_tile.HasField('tile'))
self.assertTrue(nexus_tile.tile.HasField('grid_tile'))
self.assertEqual(1, len(nexus_tile.tile.grid_tile.meta_data))
tile = nexus_tile.tile.grid_tile
self.assertEqual(38, from_shaped_array(tile.latitude).size)
self.assertEqual(87, from_shaped_array(tile.longitude).size)
self.assertEqual((1, 38, 87),
from_shaped_array(tile.variable_data).shape)
tile1_data = np.ma.masked_invalid(
from_shaped_array(results[0].tile.grid_tile.variable_data))
self.assertEqual(3306, np.ma.count(tile1_data))
self.assertAlmostEqual(
-78.375,
np.ma.min(
np.ma.masked_invalid(
from_shaped_array(results[0].tile.grid_tile.latitude))),
places=3)
self.assertAlmostEqual(
-69.125,
np.ma.max(
np.ma.masked_invalid(
from_shaped_array(results[0].tile.grid_tile.latitude))),
places=3)
self.assertEqual(1451606400, results[0].tile.grid_tile.time)
def test_process(self):
processor = GenerateTileId()
tile = nexusproto.NexusTile()
tile.summary.granule = 'test_dir/test_granule.nc'
tile.summary.data_var_name = 'test_variable'
tile.summary.section_spec = 'i:0:90,j:0:90,k:8:9,nv:0:2,tile:4:5,time:8:9'
expected_id = uuid.uuid3(
uuid.NAMESPACE_DNS, 'test_granule.nc' + 'test_variable' +
'i:0:90,j:0:90,k:8:9,nv:0:2,tile:4:5,time:8:9')
self.assertEqual(str(expected_id),
processor.process(tile).summary.tile_id)
def test_build_solr_doc_no_standard_name(self):
"""
When TileSummary.standard_name isn't available, the solr field
tile_var_name_s should use TileSummary.data_var_name
"""
tile = nexusproto.NexusTile()
tile.summary.tile_id = 'test_id'
tile.summary.data_var_name = 'test_variable'
tile.tile.ecco_tile.depth = 10.5
metadata_store = SolrStore()
solr_doc = metadata_store._build_solr_doc(tile)
self.assertEqual('test_variable', solr_doc['tile_var_name_s'])
def process(self, tile, dataset: xr.Dataset, *args, **kwargs):
try:
dimensions_to_slices = self._convert_spec_to_slices(
tile.summary.section_spec)
output_tile = nexusproto.NexusTile()
output_tile.CopyFrom(tile)
output_tile.summary.data_var_name = self.variable
return self._generate_tile(dataset, dimensions_to_slices,
output_tile)
except Exception as e:
raise TileProcessingError(
f"Could not generate tiles from the granule because of the following error: {e}."
)
def test_extract_timestamp_timeseries_exception(self):
test_file = path.join(path.dirname(__file__), 'datafiles',
'not_empty_modis.nc')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "time:0:1,lat:0:10,lon:0:10"
input_tile.summary.CopyFrom(tile_summary)
input_tile.tile.time_series_tile.CopyFrom(nexusproto.TimeSeriesTile())
with self.assertRaises(BadTimestampExtractionException):
list(self.module.process_nexus_tile(
input_tile))[0].tile.time_series_tile.time
def test_extract_timestamp_from_metadata(self):
test_file = path.join(path.dirname(__file__), 'datafiles',
'not_empty_modis.nc')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "time:0:1,lat:0:10,lon:0:10"
input_tile.summary.CopyFrom(tile_summary)
input_tile.tile.grid_tile.CopyFrom(nexusproto.GridTile())
results = list(self.module.process_nexus_tile(input_tile))
nexus_tile_after = results[0]
self.assertEqual(1537428301, nexus_tile_after.tile.grid_tile.time)
def test_read_not_empty_mur(self):
test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_mur.nc4')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "time:0:1,lat:0:10,lon:0:10"
input_tile.summary.CopyFrom(tile_summary)
results = list(self.module.process(input_tile))
self.assertEqual(1, len(results))
tile1_data = np.ma.masked_invalid(from_shaped_array(results[0].tile.grid_tile.variable_data))
self.assertEqual((1, 10, 10), tile1_data.shape)
self.assertEqual(100, np.ma.count(tile1_data))
def test_build_solr_doc(self):
tile = nexusproto.NexusTile()
tile.summary.tile_id = 'test_id'
tile.summary.dataset_name = 'test_dataset'
tile.summary.dataset_uuid = 'test_dataset_id'
tile.summary.data_var_name = 'test_variable'
tile.summary.granule = 'test_granule_path'
tile.summary.section_spec = 'time:0:1,j:0:20,i:200:240'
tile.summary.bbox.lat_min = -180.1
tile.summary.bbox.lat_max = 180.2
tile.summary.bbox.lon_min = -90.5
tile.summary.bbox.lon_max = 90.0
tile.summary.stats.min = -10.0
tile.summary.stats.max = 25.5
tile.summary.stats.mean = 12.5
tile.summary.stats.count = 100
tile.summary.stats.min_time = 694224000
tile.summary.stats.max_time = 694310400
tile.summary.standard_name = 'sea_surface_temperature'
tile.tile.ecco_tile.depth = 10.5
metadata_store = SolrStore()
solr_doc = metadata_store._build_solr_doc(tile)
self.assertEqual('sea_surface_temp', solr_doc['table_s'])
self.assertEqual(
'POLYGON((-90.500 -180.100, 90.000 -180.100, 90.000 180.200, -90.500 180.200, -90.500 -180.100))',
solr_doc['geo'])
self.assertEqual('test_id', solr_doc['id'])
self.assertEqual('test_dataset!test_id', solr_doc['solr_id_s'])
self.assertEqual('time:0:1,j:0:20,i:200:240',
solr_doc['sectionSpec_s'])
self.assertEqual('test_granule_path', solr_doc['granule_s'])
self.assertEqual('sea_surface_temperature',
solr_doc['tile_var_name_s'])
self.assertAlmostEqual(-90.5, solr_doc['tile_min_lon'])
self.assertAlmostEqual(90.0, solr_doc['tile_max_lon'])
self.assertAlmostEqual(-180.1, solr_doc['tile_min_lat'], delta=1E-5)
self.assertAlmostEqual(180.2, solr_doc['tile_max_lat'], delta=1E-5)
self.assertEqual('1992-01-01T00:00:00Z', solr_doc['tile_min_time_dt'])
self.assertEqual('1992-01-02T00:00:00Z', solr_doc['tile_max_time_dt'])
self.assertAlmostEqual(-10.0, solr_doc['tile_min_val_d'])
self.assertAlmostEqual(25.5, solr_doc['tile_max_val_d'])
self.assertAlmostEqual(12.5, solr_doc['tile_avg_val_d'])
self.assertEqual(100, solr_doc['tile_count_i'])
self.assertAlmostEqual(10.5, solr_doc['tile_depth'])
def test_read_not_empty_wswm(self):
reading_processor = TimeSeriesReadingProcessor('Qout', 'lat', 'lon', time='time')
granule_path = path.join(path.dirname(__file__), '../granules/not_empty_wswm.nc')
input_tile = nexusproto.NexusTile()
input_tile.summary.granule = granule_path
dimensions_to_slices = {
'time': slice(0, 5832),
'rivid': slice(0, 1),
}
with xr.open_dataset(granule_path) as ds:
output_tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile)
self.assertEqual(granule_path, output_tile.summary.granule, granule_path)
self.assertEqual([5832], output_tile.tile.time_series_tile.time.shape)
self.assertEqual([5832, 1], output_tile.tile.time_series_tile.variable_data.shape)
self.assertEqual([1], output_tile.tile.time_series_tile.latitude.shape)
self.assertEqual([1], output_tile.tile.time_series_tile.longitude.shape)
def test_run_chain_promote_var(self):
processor_list = [{
'name': 'GridReadingProcessor',
'config': {
'latitude': 'lat',
'longitude': 'lon',
'time': 'time',
'variable_to_read': 'analysed_sst'
}
}, {
'name': 'EmptyTileFilter',
'config': {}
}, {
'name': 'KelvinToCelsius',
'config': {}
}, {
'name': 'PromoteVariableToGlobalAttribute',
'config': {
'attribute_name': 'time_i',
'variable_name': 'time',
'dimensioned_by.0': 'time'
}
}, {
'name': 'TileSummarizingProcessor',
'config': {}
}]
processorchain = ProcessorChain(processor_list)
test_file = path.join(path.dirname(__file__), 'datafiles',
'partial_empty_mur.nc4')
input_tile = nexusproto.NexusTile()
tile_summary = nexusproto.TileSummary()
tile_summary.granule = "file:%s" % test_file
tile_summary.section_spec = "time:0:1,lat:489:499,lon:0:10"
input_tile.summary.CopyFrom(tile_summary)
results = list(processorchain.process(input_tile))
self.assertEqual(1, len(results))
tile = results[0]
self.assertEqual("1104483600",
tile.summary.global_attributes[0].values[0])
def read_tile(self):
reading_processor = GridReadingProcessor('B03',
'lat',
'lon',
time='time')
granule_path = path.join(
path.dirname(__file__),
'../granules/HLS.S30.T11SPC.2020001.v1.4.hdf.nc')
input_tile = nexusproto.NexusTile()
input_tile.summary.granule = granule_path
dimensions_to_slices = {
'time': slice(0, 1),
'lat': slice(0, 30),
'lon': slice(0, 30)
}
with xr.open_dataset(granule_path) as ds:
return reading_processor._generate_tile(ds, dimensions_to_slices,
input_tile)