def test_process_inputs_replace_multiple_zarr(self): status, output = gen_cube_wrapper([ get_inputdata_path( '20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'), get_inputdata_path( '20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'), get_inputdata_path( '20170103-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'), get_inputdata_path( '20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc') ], 'l2c.zarr', no_sort_mode=True) self.assertEqual(True, status) self.assertTrue( '\nstep 8 of 8: creating input slice in l2c.zarr...\n' in output) self.assertTrue( '\nstep 8 of 8: appending input slice to l2c.zarr...\n' in output) self.assertTrue( '\nstep 8 of 8: replacing input slice at index 1 in l2c.zarr...\n' in output) self.assert_cube_ok( xr.open_zarr('l2c.zarr'), 3, dict(date_modified=None, time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-03T12:00:00.000000000'))
def test_input_txt(self): f = open((os.path.join(os.path.dirname(__file__), 'inputdata', "input.txt")), "w+") for i in range(1, 4): file_name = f"2017010{i}-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc" file = get_inputdata_path(file_name) f.write("%s\n" % file) f.close() status, output = gen_cube_wrapper([get_inputdata_path('input.txt')], 'l2c.zarr', no_sort_mode=False) self.assertEqual(True, status) self.assert_cube_ok( xr.open_zarr('l2c.zarr'), 3, dict(time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-03T12:00:00.000000000'))
def clean_up(): files = [ 'l2c-single.nc', 'l2c-single.zarr', 'l2c.nc', 'l2c.zarr', 'l2c_1x80x60.zarr', 'l2c_1x80x80.zarr' ] for file in files: rimraf(file) rimraf(file + '.temp.nc') # May remain from Netcdf4DatasetIO.append() rimraf(get_inputdata_path("input.txt"))
def test_process_compressed_zarr(self): status, output = gen_cube_wrapper([ get_inputdata_path( '201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc') ], 'l2c.zarr', no_sort_mode=False) self.assertEqual(True, status) self.assert_cube_ok( xr.open_zarr('l2c.zarr'), expected_time_dim=3, expected_extra_attrs=dict( time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-03T12:00:00.000000000')) status, output = gen_cube_wrapper([ get_inputdata_path( '201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc') ], 'l2c_packed.zarr', output_writer_params={ 'compressor': { 'cname': 'zstd', 'clevel': 1, 'shuffle': 2 } }, no_sort_mode=False) self.assertEqual(True, status) self.assert_cube_ok( xr.open_zarr('l2c_packed.zarr'), expected_time_dim=3, expected_extra_attrs=dict( time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-03T12:00:00.000000000')) ds_default_compressor = xr.open_zarr('l2c.zarr') ds_compressed = xr.open_zarr('l2c_packed.zarr') np.testing.assert_allclose(ds_default_compressor.analysed_sst.values, ds_compressed.analysed_sst.values) self.assertEqual( "Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0)", str(ds_default_compressor.analysed_sst.encoding['compressor'])) self.assertEqual( "Blosc(cname='zstd', clevel=1, shuffle=BITSHUFFLE, blocksize=0)", str(ds_compressed.analysed_sst.encoding['compressor']))
def test_handle_360_lon(self): status, output = gen_cube_wrapper([ get_inputdata_path( '20170101120000-UKMO-L4_GHRSST-SSTfnd-OSTIAanom-GLOB-v02.0-fv02.0.nc' ) ], 'l2c-single.zarr', no_sort_mode=False) self.assertEqual(True, status) ds = xr.open_zarr('l2c-single.zarr') self.assertIn('lon', ds.coords) self.assertFalse(np.any(ds.coords['lon'] > 180.))
def test_process_inputs_single_zarr(self): status, output = gen_cube_wrapper([ get_inputdata_path( '20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc') ], 'l2c-single.zarr') self.assertEqual(True, status) self.assertTrue( '\nstep 8 of 8: creating input slice in l2c-single.zarr...\n' in output) self.assert_cube_ok( xr.open_zarr('l2c-single.zarr'), 1, dict(date_modified=None, time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-01T12:00:00.000000000'))
def test_illegal_proc(self): with self.assertRaises(ValueError) as e: gen_cube_wrapper([ get_inputdata_path( '20170101120000-UKMO-L4_GHRSST-SSTfnd-OSTIAanom-GLOB-v02.0-fv02.0.nc' ) ], 'l2c-single.zarr', no_sort_mode=False, input_processor_name="") self.assertEqual('input_processor_name must not be empty', f'{e.exception}') with self.assertRaises(ValueError) as e: gen_cube_wrapper([ get_inputdata_path( '20170101120000-UKMO-L4_GHRSST-SSTfnd-OSTIAanom-GLOB-v02.0-fv02.0.nc' ) ], 'l2c-single.zarr', no_sort_mode=False, input_processor_name='chris-proba') self.assertEqual("Unknown input_processor_name 'chris-proba'", f'{e.exception}')
def test_process_inputs_single_nc(self): status, output = gen_cube_wrapper([ get_inputdata_path( '20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc') ], 'l2c-single.nc') self.assertEqual(True, status) self.assertTrue( '\nstep 9 of 9: creating input slice in l2c-single.nc...\n' in output) self.assert_cube_ok( xr.open_dataset('l2c-single.nc', autoclose=True), expected_time_dim=1, expected_extra_attrs=dict( date_modified=None, time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-01T12:00:00.000000000'))
def test_process_inputs_append_multiple_nc(self): status, output = gen_cube_wrapper([ get_inputdata_path( '201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc') ], 'l2c.nc', no_sort_mode=False) self.assertEqual(True, status) self.assertTrue( '\nstep 8 of 8: creating input slice in l2c.nc...\n' in output) self.assertTrue( '\nstep 8 of 8: appending input slice to l2c.nc...\n' in output) self.assert_cube_ok( xr.open_dataset('l2c.nc', autoclose=True), 3, dict(date_modified=None, time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-03T12:00:00.000000000'))
def test_process_inputs_append_multiple_zarr(self): status, output = gen_cube_wrapper([ get_inputdata_path( '201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc') ], 'l2c.zarr', no_sort_mode=False) self.assertEqual(True, status) self.assertTrue( '\nstep 9 of 9: creating input slice in l2c.zarr...\n' in output) self.assertTrue( '\nstep 9 of 9: appending input slice to l2c.zarr...\n' in output) self.assert_cube_ok( xr.open_zarr('l2c.zarr'), expected_time_dim=3, expected_extra_attrs=dict( date_modified=None, time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-03T12:00:00.000000000'))
def test_process_inputs_insert_multiple_zarr(self): status, output = gen_cube_wrapper([ get_inputdata_path( '20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'), get_inputdata_path( '20170103-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'), get_inputdata_path( '20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc') ], 'l2c_presorted.zarr') self.assertEqual(True, status) self.assertTrue( '\nstep 9 of 9: creating input slice in l2c_presorted.zarr...\n' in output) self.assertTrue( '\nstep 9 of 9: appending input slice to l2c_presorted.zarr...\n' in output) self.assertFalse( '\nstep 9 of 9: inserting input slice before index 0 in l2c_presorted.zarr...\n' in output) status, output = gen_cube_wrapper([ get_inputdata_path( '20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'), get_inputdata_path( '20170103-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc'), get_inputdata_path( '20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc') ], 'l2c.zarr', no_sort_mode=True) self.assertEqual(True, status) self.assertTrue( '\nstep 9 of 9: creating input slice in l2c.zarr...\n' in output) self.assertTrue( '\nstep 9 of 9: appending input slice to l2c.zarr...\n' in output) self.assertTrue( '\nstep 9 of 9: inserting input slice before index 0 in l2c.zarr...\n' in output) self.assert_cube_ok( xr.open_zarr('l2c.zarr'), expected_time_dim=3, expected_extra_attrs=dict( date_modified=None, time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-03T12:00:00.000000000')) ds_presorted = xr.open_zarr('l2c_presorted.zarr') ds_insert = xr.open_zarr('l2c.zarr') np.testing.assert_allclose(ds_presorted.analysed_sst.values, ds_insert.analysed_sst.values)
def test_process_inputs_single_nc_processed_vars(self): status, output = gen_cube_wrapper( [ get_inputdata_path( '20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc' ) ], 'l2c-single.nc', processed_variables=( ('analysed_sst', dict(valid_pixel_expression=None)), ('analysis_error', dict(valid_pixel_expression=None)), ('sea_ice_fraction', dict(valid_pixel_expression=None)), ('water_mask', dict(expression='(mask.sea or mask.lake) and not mask.ice', load=True)), ('ice_mask', dict(expression='mask.sea and mask.ice')), ('analysed_sst', dict(valid_pixel_expression='water_mask')), ('analysis_error', dict(valid_pixel_expression='water_mask')), ('sea_ice_fraction', dict(valid_pixel_expression='ice_mask')), ), output_variables=( ('analysed_sst', dict(name='SST')), ('analysis_error', dict(name='SST_uncertainty')), ('sea_ice_fraction', None), ), ) self.assertEqual(True, status) self.assertTrue( '\nstep 9 of 9: creating input slice in l2c-single.nc...\n' in output) with xr.open_dataset('l2c-single.nc') as dataset: self.assert_cube_ok( dataset, expected_time_dim=1, expected_output_vars=('SST', 'SST_uncertainty', 'sea_ice_fraction'), expected_extra_attrs=dict( date_modified=None, time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-01T12:00:00.000000000'))
def test_process_chunked_zarr(self): status, output = gen_cube_wrapper([ get_inputdata_path( '201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc') ], 'l2c.zarr', no_sort_mode=False) self.assertEqual(True, status) self.assert_cube_ok( xr.open_zarr('l2c.zarr'), expected_time_dim=3, expected_extra_attrs=dict( time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-03T12:00:00.000000000')) status, output = gen_cube_wrapper([ get_inputdata_path( '201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc') ], 'l2c_1x80x60.zarr', output_writer_params={ 'chunksizes': { 'lon': 80, 'lat': 60 } }, no_sort_mode=False) self.assertEqual(True, status) self.assert_cube_ok( xr.open_zarr('l2c_1x80x60.zarr'), expected_time_dim=3, expected_extra_attrs=dict( time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-03T12:00:00.000000000')) ds_unchunked = xr.open_zarr('l2c.zarr') ds_chunked = xr.open_zarr('l2c_1x80x60.zarr') np.testing.assert_allclose(ds_unchunked.analysed_sst.values, ds_chunked.analysed_sst.values) self.assertEqual(((1, 1, 1), (60, 60, 60), (80, 80, 80, 80)), ds_chunked.analysed_sst.chunks) status, output = gen_cube_wrapper([ get_inputdata_path( '201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc') ], 'l2c_1x80x80.zarr', output_writer_params={ 'chunksizes': { 'lon': 80, 'lat': 80 } }, no_sort_mode=False) self.assertEqual(True, status) self.assert_cube_ok( xr.open_zarr('l2c_1x80x80.zarr'), expected_time_dim=3, expected_extra_attrs=dict( time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-03T12:00:00.000000000')) ds_unchunked = xr.open_zarr('l2c.zarr') ds_chunked = xr.open_zarr('l2c_1x80x80.zarr') np.testing.assert_allclose(ds_unchunked.analysed_sst.values, ds_chunked.analysed_sst.values) self.assertEqual(((1, 1, 1), (80, 80, 20), (80, 80, 80, 80)), ds_chunked.analysed_sst.chunks)
def test_replace_packed_zarr(self): status, output = gen_cube_wrapper( [ get_inputdata_path( '20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc' ), get_inputdata_path( '20170103-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc' ), get_inputdata_path( '20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc' ) ], 'l2c_presorted.zarr', output_writer_params={ 'chunksizes': { 'lon': 80, 'lat': 80 }, 'packing': { 'analysed_sst': { 'scale_factor': 0.07324442274239326, 'add_offset': -300.0, 'dtype': 'uint16', '_FillValue': 65535 } } }) self.assertEqual(True, status) self.assertTrue( '\nstep 9 of 9: creating input slice in l2c_presorted.zarr...\n' in output) self.assertTrue( '\nstep 9 of 9: appending input slice to l2c_presorted.zarr...\n' in output) self.assertFalse( '\nstep 9 of 9: inserting input slice before index 0 in l2c_presorted.zarr...\n' in output) status, output = gen_cube_wrapper( [ get_inputdata_path( '20170101-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc' ), get_inputdata_path( '20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc' ), get_inputdata_path( '20170103-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc' ), get_inputdata_path( '20170102-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc' ) ], 'l2c.zarr', output_writer_params={ 'chunksizes': { 'lon': 80, 'lat': 80 }, 'packing': { 'analysed_sst': { 'scale_factor': 0.07324442274239326, 'add_offset': -300.0, 'dtype': 'uint16', '_FillValue': 65535 } } }, no_sort_mode=True) self.assertEqual(True, status) self.assertTrue( '\nstep 9 of 9: creating input slice in l2c.zarr...\n' in output) self.assertTrue( '\nstep 9 of 9: appending input slice to l2c.zarr...\n' in output) self.assertTrue( '\nstep 9 of 9: replacing input slice at index 1 in l2c.zarr...\n' in output) self.assert_cube_ok( xr.open_zarr('l2c.zarr'), expected_time_dim=3, expected_extra_attrs=dict( date_modified=None, time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-03T12:00:00.000000000')) ds_presorted = xr.open_zarr('l2c_presorted.zarr') ds_insert = xr.open_zarr('l2c.zarr') np.testing.assert_allclose(ds_presorted.analysed_sst.values, ds_insert.analysed_sst.values)
def test_process_packed_zarr(self): status, output = gen_cube_wrapper([ get_inputdata_path( '201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc') ], 'l2c.zarr', no_sort_mode=False) self.assertEqual(True, status) self.assert_cube_ok( xr.open_zarr('l2c.zarr'), expected_time_dim=3, expected_extra_attrs=dict( time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-03T12:00:00.000000000')) status, output = gen_cube_wrapper( [ get_inputdata_path( '201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc' ) ], 'l2c_packed.zarr', output_writer_params={ 'packing': { 'analysed_sst': { 'scale_factor': 0.07324442274239326, 'add_offset': -300.0, 'dtype': 'uint16', '_FillValue': 65535 } } }, no_sort_mode=False) self.assertEqual(True, status) self.assert_cube_ok( xr.open_zarr('l2c_packed.zarr'), expected_time_dim=3, expected_extra_attrs=dict( time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-03T12:00:00.000000000')) ds_unpacked = xr.open_zarr('l2c.zarr') ds_packed = xr.open_zarr('l2c_packed.zarr') np.testing.assert_almost_equal(ds_unpacked.analysed_sst.values, ds_packed.analysed_sst.values, decimal=1) self.assertEqual(0.07324442274239326, ds_packed.analysed_sst.encoding['scale_factor']) self.assertEqual(-300.0, ds_packed.analysed_sst.encoding['add_offset']) self.assertEqual('uint16', ds_packed.analysed_sst.encoding['dtype']) self.assertEqual(65535, ds_packed.analysed_sst.encoding['_FillValue']) self.assertEqual((1, 180, 320), ds_packed.analysed_sst.encoding['chunks']) status, output = gen_cube_wrapper( [ get_inputdata_path( '201701??-IFR-L4_GHRSST-SSTfnd-ODYSSEA-NWE_002-v2.0-fv1.0.nc' ) ], 'l2c_packed_1x80x80.zarr', output_writer_params={ 'chunksizes': { 'lon': 80, 'lat': 80 }, 'packing': { 'analysed_sst': { 'scale_factor': 0.07324442274239326, 'add_offset': -300.0, 'dtype': 'uint16', '_FillValue': 65535 } } }, no_sort_mode=False) self.assertEqual(True, status) self.assert_cube_ok( xr.open_zarr('l2c_packed_1x80x80.zarr'), expected_time_dim=3, expected_extra_attrs=dict( time_coverage_start='2016-12-31T12:00:00.000000000', time_coverage_end='2017-01-03T12:00:00.000000000')) ds_unpacked = xr.open_zarr('l2c.zarr') ds_packed = xr.open_zarr('l2c_packed_1x80x80.zarr') np.testing.assert_almost_equal(ds_unpacked.analysed_sst.values, ds_packed.analysed_sst.values, decimal=1) self.assertEqual(((1, 1, 1), (80, 80, 20), (80, 80, 80, 80)), ds_packed.analysed_sst.chunks) self.assertEqual(0.07324442274239326, ds_packed.analysed_sst.encoding['scale_factor']) self.assertEqual(-300.0, ds_packed.analysed_sst.encoding['add_offset']) self.assertEqual('uint16', ds_packed.analysed_sst.encoding['dtype']) self.assertEqual(65535, ds_packed.analysed_sst.encoding['_FillValue']) self.assertEqual((1, 80, 80), ds_packed.analysed_sst.encoding['chunks'])