def test_init(self): """Test basic init method of writer.""" from satpy.writers.awips_tiled import AWIPSTiledWriter AWIPSTiledWriter(base_dir=self.base_dir)
def test_lettered_tiles_update_existing(self): """Test updating lettered tiles with additional data.""" import shutil import dask import xarray as xr from satpy.writers.awips_tiled import AWIPSTiledWriter first_base_dir = os.path.join(self.base_dir, 'first') w = AWIPSTiledWriter(base_dir=first_base_dir, compress=True) shape = (2000, 1000) data = np.linspace(0., 1., shape[0] * shape[1], dtype=np.float32).reshape(shape) # pixels to be filled in later data[:, -200:] = np.nan data = da.from_array(data, chunks=500) area_def = self._get_test_area(shape=(2000, 1000), extents=(-1000000., -1500000., 1000000., 1500000.)) ds = self._get_test_lcc_data(data, area_def) # tile_count should be ignored since we specified lettered_grid w.save_datasets([ds], sector_id='LCC', source_name="TESTS", tile_count=(3, 3), lettered_grid=True) all_files = sorted(glob(os.path.join(first_base_dir, 'TESTS_AII*.nc'))) assert len(all_files) == 16 first_files = [] second_base_dir = os.path.join(self.base_dir, 'second') os.makedirs(second_base_dir) for fn in all_files: new_fn = fn.replace(first_base_dir, second_base_dir) shutil.copy(fn, new_fn) first_files.append(new_fn) # Second writing/updating # Area is about 100 pixels to the right area_def2 = self._get_test_area(shape=(2000, 1000), extents=(-800000., -1500000., 1200000., 1500000.)) data2 = np.linspace(0., 1., 2000000, dtype=np.float32).reshape( (2000, 1000)) # a gap at the beginning where old values remain data2[:, :200] = np.nan # a gap at the end where old values remain data2[:, -400:-300] = np.nan data2 = da.from_array(data2, chunks=500) ds2 = self._get_test_lcc_data(data2, area_def2) w = AWIPSTiledWriter(base_dir=second_base_dir, compress=True) # HACK: The _copy_to_existing function hangs when opening the output # file multiple times...sometimes. If we limit dask to one worker # it seems to work fine. with dask.config.set(num_workers=1): w.save_datasets([ds2], sector_id='LCC', source_name="TESTS", tile_count=(3, 3), lettered_grid=True) all_files = glob(os.path.join(second_base_dir, 'TESTS_AII*.nc')) # 16 original tiles + 4 new tiles assert len(all_files) == 20 # these tiles should be the right-most edge of the first image first_right_edge_files = [ x for x in first_files if 'P02' in x or 'P04' in x or 'V02' in x or 'V04' in x ] for new_file in first_right_edge_files: orig_file = new_file.replace(second_base_dir, first_base_dir) orig_nc = xr.open_dataset(orig_file) orig_data = orig_nc['data'].values if not np.isnan(orig_data).any(): # we only care about the tiles that had NaNs originally continue new_nc = xr.open_dataset(new_file) new_data = new_nc['data'].values # there should be at least some areas of the file # that old data was present and hasn't been replaced np.testing.assert_allclose(orig_data[:, :20], new_data[:, :20]) # it isn't exactly 200 because the tiles aren't aligned with the # data (the left-most tile doesn't have data until some columns # in), but it should be at least that many columns assert np.isnan(orig_data[:, 200:]).all() assert not np.isnan(new_data[:, 200:]).all()
def test_multivar_numbered_tiles_glm(self, sector): """Test creating a tiles with multiple variables.""" import xarray as xr from satpy.writers.awips_tiled import AWIPSTiledWriter from xarray import DataArray from pyresample.geometry import AreaDefinition from pyresample.utils import proj4_str_to_dict w = AWIPSTiledWriter(base_dir=self.base_dir, compress=True) area_def = AreaDefinition( 'test', 'test', 'test', proj4_str_to_dict( '+proj=lcc +datum=WGS84 +ellps=WGS84 +lon_0=-95. ' '+lat_0=25 +lat_1=25 +units=m +no_defs'), 100, 200, (-1000., -1500., 1000., 1500.), ) now = datetime(2018, 1, 1, 12, 0, 0) end_time = now + timedelta(minutes=20) ds1 = DataArray(da.from_array(np.linspace(0., 1., 20000, dtype=np.float32).reshape( (200, 100)), chunks=50), attrs=dict(name='total_energy', platform_name='GOES-17', sensor='SENSOR', units='1', area=area_def, start_time=now, end_time=end_time, scan_mode='M3', scene_abbr=sector, platform_shortname="G17")) ds2 = ds1.copy() ds2.attrs.update({ 'name': 'flash_extent_density', }) ds3 = ds1.copy() ds3.attrs.update({ 'name': 'average_flash_area', }) dqf = ds1.copy() dqf = (dqf * 255).astype(np.uint8) dqf.attrs = ds1.attrs.copy() dqf.attrs.update({ 'name': 'DQF', '_FillValue': 1, }) w.save_datasets([ds1, ds2, ds3, dqf], sector_id='TEST', source_name="TESTS", tile_count=(3, 3), template='glm_l2_rad{}'.format(sector.lower())) all_files = glob(os.path.join(self.base_dir, '*_GLM*.nc')) assert len(all_files) == 9 for fn in all_files: ds = xr.open_dataset(fn, mask_and_scale=False) check_required_common_attributes(ds) if sector == 'C': assert ds.attrs['time_coverage_end'] == end_time.strftime( '%Y-%m-%dT%H:%M:%S.%fZ') else: # 'F' assert ds.attrs['time_coverage_end'] == end_time.strftime( '%Y-%m-%dT%H:%M:%SZ')