def test_save_mp4_distributed(self): """Save a series of fake scenes to an mp4 video.""" from satpy import MultiScene area = _create_test_area() scenes = _create_test_scenes(area=area) # Add a dataset to only one of the Scenes scenes[1]['ds3'] = _create_test_dataset('ds3') # Add a start and end time for ds_id in ['ds1', 'ds2', 'ds3']: scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) if ds_id == 'ds3': continue scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) mscn = MultiScene(scenes) fn = os.path.join( self.base_dir, 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') writer_mock = mock.MagicMock() client_mock = mock.MagicMock() client_mock.compute.side_effect = lambda x: tuple(v.compute() for v in x) client_mock.gather.side_effect = lambda x: x with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: get_writer.return_value = writer_mock # force order of datasets by specifying them mscn.save_animation(fn, client=client_mock, datasets=['ds1', 'ds2', 'ds3']) # 2 saves for the first scene + 1 black frame # 3 for the second scene self.assertEqual(writer_mock.append_data.call_count, 3 + 3) filenames = [os.path.basename(args[0][0]) for args in get_writer.call_args_list] self.assertEqual(filenames[0], 'test_save_mp4_ds1_20180101_00_20180102_12.mp4') self.assertEqual(filenames[1], 'test_save_mp4_ds2_20180101_00_20180102_12.mp4') self.assertEqual(filenames[2], 'test_save_mp4_ds3_20180102_00_20180102_12.mp4') # Test no distributed client found mscn = MultiScene(scenes) fn = os.path.join( self.base_dir, 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') writer_mock = mock.MagicMock() client_mock = mock.MagicMock() client_mock.compute.side_effect = lambda x: tuple(v.compute() for v in x) client_mock.gather.side_effect = lambda x: x with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer, \ mock.patch('satpy.multiscene.get_client', mock.Mock(side_effect=ValueError("No client"))): get_writer.return_value = writer_mock # force order of datasets by specifying them mscn.save_animation(fn, datasets=['ds1', 'ds2', 'ds3']) # 2 saves for the first scene + 1 black frame # 3 for the second scene self.assertEqual(writer_mock.append_data.call_count, 3 + 3) filenames = [os.path.basename(args[0][0]) for args in get_writer.call_args_list] self.assertEqual(filenames[0], 'test_save_mp4_ds1_20180101_00_20180102_12.mp4') self.assertEqual(filenames[1], 'test_save_mp4_ds2_20180101_00_20180102_12.mp4') self.assertEqual(filenames[2], 'test_save_mp4_ds3_20180102_00_20180102_12.mp4')
def test_save_datasets_simple(self): """Save a series of fake scenes to an PNG images.""" from satpy import MultiScene area = _create_test_area() scenes = _create_test_scenes(area=area) # Add a dataset to only one of the Scenes scenes[1]['ds3'] = _create_test_dataset('ds3') # Add a start and end time for ds_id in ['ds1', 'ds2', 'ds3']: scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) if ds_id == 'ds3': continue scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) mscn = MultiScene(scenes) client_mock = mock.MagicMock() client_mock.compute.side_effect = lambda x: tuple(v for v in x) client_mock.gather.side_effect = lambda x: x with mock.patch( 'satpy.multiscene.Scene.save_datasets') as save_datasets: save_datasets.return_value = [True] # some arbitrary return value # force order of datasets by specifying them mscn.save_datasets(base_dir=self.base_dir, client=False, datasets=['ds1', 'ds2', 'ds3'], writer='simple_image') # 2 for each scene self.assertEqual(save_datasets.call_count, 2)
def test_properties(self): """Test basic properties/attributes of the MultiScene.""" from satpy import MultiScene area = _create_test_area() scenes = _create_test_scenes(area=area) ds1_id = make_dataid(name='ds1') ds2_id = make_dataid(name='ds2') ds3_id = make_dataid(name='ds3') ds4_id = make_dataid(name='ds4') # Add a dataset to only one of the Scenes scenes[1]['ds3'] = _create_test_dataset('ds3') mscn = MultiScene(scenes) self.assertSetEqual(mscn.loaded_dataset_ids, {ds1_id, ds2_id, ds3_id}) self.assertSetEqual(mscn.shared_dataset_ids, {ds1_id, ds2_id}) self.assertTrue(mscn.all_same_area) bigger_area = _create_test_area(shape=(20, 40)) scenes[0]['ds4'] = _create_test_dataset('ds4', shape=(20, 40), area=bigger_area) self.assertSetEqual(mscn.loaded_dataset_ids, {ds1_id, ds2_id, ds3_id, ds4_id}) self.assertSetEqual(mscn.shared_dataset_ids, {ds1_id, ds2_id}) self.assertFalse(mscn.all_same_area)
def test_save_datasets_distributed_source_target(self): """Test distributed save for writers returning sources and targets e.g. geotiff writer.""" from satpy import MultiScene import dask.array as da area = _create_test_area() scenes = _create_test_scenes(area=area) # Add a dataset to only one of the Scenes scenes[1]['ds3'] = _create_test_dataset('ds3') # Add a start and end time for ds_id in ['ds1', 'ds2', 'ds3']: scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) if ds_id == 'ds3': continue scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) mscn = MultiScene(scenes) client_mock = mock.MagicMock() client_mock.compute.side_effect = lambda x: tuple(v for v in x) client_mock.gather.side_effect = lambda x: x source_mock = mock.MagicMock() source_mock.__class__ = da.Array target_mock = mock.MagicMock() with mock.patch( 'satpy.multiscene.Scene.save_datasets') as save_datasets: save_datasets.return_value = [(source_mock, target_mock) ] # some arbitrary return value # force order of datasets by specifying them with self.assertRaises(NotImplementedError): mscn.save_datasets(base_dir=self.base_dir, client=client_mock, datasets=['ds1', 'ds2', 'ds3'], writer='geotiff')
def test_crop(self): """Test the crop method.""" import numpy as np from pyresample.geometry import AreaDefinition from xarray import DataArray from satpy import MultiScene, Scene scene1 = Scene() area_extent = (-5570248.477339745, -5561247.267842293, 5567248.074173927, 5570248.477339745) proj_dict = { 'a': 6378169.0, 'b': 6356583.8, 'h': 35785831.0, 'lon_0': 0.0, 'proj': 'geos', 'units': 'm' } x_size = 3712 y_size = 3712 area_def = AreaDefinition( 'test', 'test', 'test', proj_dict, x_size, y_size, area_extent, ) area_def2 = AreaDefinition( 'test2', 'test2', 'test2', proj_dict, x_size // 2, y_size // 2, area_extent, ) scene1["1"] = DataArray(np.zeros((y_size, x_size))) scene1["2"] = DataArray(np.zeros((y_size, x_size)), dims=('y', 'x')) scene1["3"] = DataArray(np.zeros((y_size, x_size)), dims=('y', 'x'), attrs={'area': area_def}) scene1["4"] = DataArray(np.zeros((y_size // 2, x_size // 2)), dims=('y', 'x'), attrs={'area': area_def2}) mscn = MultiScene([scene1]) # by lon/lat bbox new_mscn = mscn.crop(ll_bbox=(-20., -5., 0, 0)) new_scn1 = list(new_mscn.scenes)[0] self.assertIn('1', new_scn1) self.assertIn('2', new_scn1) self.assertIn('3', new_scn1) self.assertTupleEqual(new_scn1['1'].shape, (y_size, x_size)) self.assertTupleEqual(new_scn1['2'].shape, (y_size, x_size)) self.assertTupleEqual(new_scn1['3'].shape, (184, 714)) self.assertTupleEqual(new_scn1['4'].shape, (92, 357))
def test_save_mp4(self): """Save a series of fake scenes to an mp4 video.""" from satpy import MultiScene area = _create_test_area() scenes = _create_test_scenes(area=area) # Add a dataset to only one of the Scenes scenes[1]['ds3'] = _create_test_dataset('ds3') # Add a start and end time for ds_id in ['ds1', 'ds2', 'ds3']: scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) if ds_id == 'ds3': continue scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) mscn = MultiScene(scenes) fn = os.path.join( self.base_dir, 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') writer_mock = mock.MagicMock() with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: get_writer.return_value = writer_mock # force order of datasets by specifying them mscn.save_animation(fn, datasets=['ds1', 'ds2', 'ds3'], client=False) # 2 saves for the first scene + 1 black frame # 3 for the second scene self.assertEqual(writer_mock.append_data.call_count, 3 + 3) filenames = [os.path.basename(args[0][0]) for args in get_writer.call_args_list] self.assertEqual(filenames[0], 'test_save_mp4_ds1_20180101_00_20180102_12.mp4') self.assertEqual(filenames[1], 'test_save_mp4_ds2_20180101_00_20180102_12.mp4') self.assertEqual(filenames[2], 'test_save_mp4_ds3_20180102_00_20180102_12.mp4') # make sure that not specifying datasets still saves all of them fn = os.path.join( self.base_dir, 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') writer_mock = mock.MagicMock() with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: get_writer.return_value = writer_mock # force order of datasets by specifying them mscn.save_animation(fn, client=False) # the 'ds3' dataset isn't known to the first scene so it doesn't get saved # 2 for first scene, 2 for second scene self.assertEqual(writer_mock.append_data.call_count, 2 + 2) self.assertIn('test_save_mp4_ds1_20180101_00_20180102_12.mp4', filenames) self.assertIn('test_save_mp4_ds2_20180101_00_20180102_12.mp4', filenames) self.assertIn('test_save_mp4_ds3_20180102_00_20180102_12.mp4', filenames)
def test_group(self): from satpy import Scene, MultiScene, DatasetID ds1 = _create_test_dataset(name='ds1') ds2 = _create_test_dataset(name='ds2') ds3 = _create_test_dataset(name='ds3') ds4 = _create_test_dataset(name='ds4') scene1 = Scene() scene1['ds1'] = ds1 scene1['ds2'] = ds2 scene2 = Scene() scene2['ds3'] = ds3 scene2['ds4'] = ds4 multi_scene = MultiScene([scene1, scene2]) groups = {DatasetID(name='odd', wavelength=(1, 2, 3)): ['ds1', 'ds3'], DatasetID(name='even', wavelength=(2, 3, 4)): ['ds2', 'ds4']} multi_scene.group(groups) self.assertSetEqual(multi_scene.shared_dataset_ids, set(groups.keys()))
def test_save_datasets_distributed_delayed(self): """Test distributed save for writers returning delayed obejcts e.g. simple_image.""" from dask.delayed import Delayed from satpy import MultiScene area = _create_test_area() scenes = _create_test_scenes(area=area) # Add a dataset to only one of the Scenes scenes[1]['ds3'] = _create_test_dataset('ds3') # Add a start and end time for ds_id in ['ds1', 'ds2', 'ds3']: scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) if ds_id == 'ds3': continue scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) mscn = MultiScene(scenes) client_mock = mock.MagicMock() client_mock.compute.side_effect = lambda x: tuple(v for v in x) client_mock.gather.side_effect = lambda x: x future_mock = mock.MagicMock() future_mock.__class__ = Delayed with mock.patch( 'satpy.multiscene.Scene.save_datasets') as save_datasets: save_datasets.return_value = [future_mock ] # some arbitrary return value # force order of datasets by specifying them mscn.save_datasets(base_dir=self.base_dir, client=client_mock, datasets=['ds1', 'ds2', 'ds3'], writer='simple_image') # 2 for each scene self.assertEqual(save_datasets.call_count, 2)
def test_save_mp4(smg, tmp_path): """Save a series of fake scenes to an mp4 video.""" from satpy import MultiScene area = _create_test_area() scenes = _create_test_scenes(area=area) smg.side_effect = _fake_get_enhanced_image # Add a dataset to only one of the Scenes scenes[1]['ds3'] = _create_test_dataset('ds3') # Add a start and end time for ds_id in ['ds1', 'ds2', 'ds3']: scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) if ds_id == 'ds3': continue scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) mscn = MultiScene(scenes) fn = str( tmp_path / 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') writer_mock = mock.MagicMock() with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: get_writer.return_value = writer_mock # force order of datasets by specifying them mscn.save_animation(fn, datasets=['ds1', 'ds2', 'ds3'], client=False) # 2 saves for the first scene + 1 black frame # 3 for the second scene assert writer_mock.append_data.call_count == 3 + 3 filenames = [ os.path.basename(args[0][0]) for args in get_writer.call_args_list ] assert filenames[0] == 'test_save_mp4_ds1_20180101_00_20180102_12.mp4' assert filenames[1] == 'test_save_mp4_ds2_20180101_00_20180102_12.mp4' assert filenames[2] == 'test_save_mp4_ds3_20180102_00_20180102_12.mp4' # make sure that not specifying datasets still saves all of them fn = str( tmp_path / 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') writer_mock = mock.MagicMock() with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: get_writer.return_value = writer_mock # force order of datasets by specifying them mscn.save_animation(fn, client=False) # the 'ds3' dataset isn't known to the first scene so it doesn't get saved # 2 for first scene, 2 for second scene assert writer_mock.append_data.call_count == 2 + 2 assert "test_save_mp4_ds1_20180101_00_20180102_12.mp4" in filenames assert "test_save_mp4_ds2_20180101_00_20180102_12.mp4" in filenames assert "test_save_mp4_ds3_20180102_00_20180102_12.mp4" in filenames # test decorating and enhancing fn = str( tmp_path / 'test-{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}-rich.mp4') writer_mock = mock.MagicMock() with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: get_writer.return_value = writer_mock mscn.save_animation( fn, client=False, enh_args={ "decorate": { "decorate": [{ "text": { "txt": "Test {start_time:%Y-%m-%d %H:%M} - " "{end_time:%Y-%m-%d %H:%M}" } }] } }) assert writer_mock.append_data.call_count == 2 + 2 assert ( "2018-01-02" in smg.call_args_list[-1][1]["decorate"]["decorate"][0]["text"]["txt"])
def multi_scene(self, scene1, scene2): """Create small multi scene for testing.""" from satpy import MultiScene return MultiScene([scene1, scene2])
def test_init_children(self): """Test creating a multiscene with children.""" from satpy import MultiScene scenes = _create_test_scenes() MultiScene(scenes)
def test_init_empty(self): """Test creating a multiscene with no children.""" from satpy import MultiScene MultiScene()
data_mon1 = glob("./data/W_XX-EUMETSAT-Darmstadt,VIS+IR+HRV+IMAGERY,MSG3+SEVIRI_C_EUMG_*.nc") data_mon2 = glob("./data/W_XX-EUMETSAT-Darmstadt,VIS+IR+HRV+IMAGERY,MSG4+SEVIRI_C_EUMG_*.nc") #filter 12am for i in data_mon1: if i[-9:-7] != "12": data_mon1.remove(i) for i in data_mon2: if i[-9:-7] != "12": data_mon2.remove(i) #load multiscene object scenes_mon1 = [Scene(reader="seviri_l1b_nc", filenames=[f]) for f in data_mon1] mscn_mon1 = MultiScene(scenes_mon1) scenes_mon2 = [Scene(reader="seviri_l1b_nc", filenames=[f]) for f in data_mon2] mscn_mon2 = MultiScene(scenes_mon2) #show all available spectral bands scenes_mon1[1].all_dataset_names() #https://sentinel.esa.int/web/sentinel/technical-guides/sentinel-2-msi/level-1c/cloud-masks #https://gisgeography.com/spectral-signature/ #load SWIR mscn_mon1.load(["IR_134"]) mscn_mon2.load(["IR_134"]) #Area definition dem. rep kongo
if time != needed_time: data_mon_1.remove(i) # data_mon_2 for i in data_mon_2: time = i[-9:-7] if time != needed_time: data_mon_2.remove(i) ## load multiscenes # data_mon_1 scenes_mon_1 = [ Scene(reader="seviri_l1b_nc", filenames=[f]) for f in data_mon_1 ] mscn_mon_1 = MultiScene(scenes_mon_1) # data_mon_21 scenes_mon_2 = [ Scene(reader="seviri_l1b_nc", filenames=[f]) for f in data_mon_2 ] mscn_mon_2 = MultiScene(scenes_mon_2) ## load the IR_134 Band scenes_mon_1[1].all_dataset_names() mscn_mon_1.load(["IR_134"]) mscn_mon_2.load(["IR_134"]) # area definition area_def_kongo = AreaDefinition( "Kongo", "A lambert azimutal equal area projection of Kongo",
import glob import os os.chdir(r"C:\Users\timhe\Documents\VIIRStest") filenames = glob.glob( "GIMGO-SVI05_npp_d20190910_t1038162_e1043566_b40776_c20190912092651273026_noac_ops*" ) global_scene = Scene(reader="viirs_sdr", filenames=filenames) nebraska_scene = global_scene.resample('nebraska', resampler='nearest') nebraska_scene.save_datasets() from satpy import MultiScene mscn = MultiScene(global_scene) mscn.load(['I05']) my_area = global_scene['I05'].load().attrs['area'].compute_optimal_bb_area({ 'proj': 'lcc', 'lon_0': -95., 'lat_0': 25., 'lat_1': 25., 'lat_2': 25. }) new_scn = global_scene.resample(my_area)