def test_init(self):
"""Test basic init method of writer."""
from satpy.writers.awips_tiled import AWIPSTiledWriter
AWIPSTiledWriter(base_dir=self.base_dir)
def test_lettered_tiles_update_existing(self):
"""Test updating lettered tiles with additional data."""
import shutil
import dask
import xarray as xr
from satpy.writers.awips_tiled import AWIPSTiledWriter
first_base_dir = os.path.join(self.base_dir, 'first')
w = AWIPSTiledWriter(base_dir=first_base_dir, compress=True)
shape = (2000, 1000)
data = np.linspace(0., 1., shape[0] * shape[1],
dtype=np.float32).reshape(shape)
# pixels to be filled in later
data[:, -200:] = np.nan
data = da.from_array(data, chunks=500)
area_def = self._get_test_area(shape=(2000, 1000),
extents=(-1000000., -1500000., 1000000.,
1500000.))
ds = self._get_test_lcc_data(data, area_def)
# tile_count should be ignored since we specified lettered_grid
w.save_datasets([ds],
sector_id='LCC',
source_name="TESTS",
tile_count=(3, 3),
lettered_grid=True)
all_files = sorted(glob(os.path.join(first_base_dir, 'TESTS_AII*.nc')))
assert len(all_files) == 16
first_files = []
second_base_dir = os.path.join(self.base_dir, 'second')
os.makedirs(second_base_dir)
for fn in all_files:
new_fn = fn.replace(first_base_dir, second_base_dir)
shutil.copy(fn, new_fn)
first_files.append(new_fn)
# Second writing/updating
# Area is about 100 pixels to the right
area_def2 = self._get_test_area(shape=(2000, 1000),
extents=(-800000., -1500000., 1200000.,
1500000.))
data2 = np.linspace(0., 1., 2000000, dtype=np.float32).reshape(
(2000, 1000))
# a gap at the beginning where old values remain
data2[:, :200] = np.nan
# a gap at the end where old values remain
data2[:, -400:-300] = np.nan
data2 = da.from_array(data2, chunks=500)
ds2 = self._get_test_lcc_data(data2, area_def2)
w = AWIPSTiledWriter(base_dir=second_base_dir, compress=True)
# HACK: The _copy_to_existing function hangs when opening the output
# file multiple times...sometimes. If we limit dask to one worker
# it seems to work fine.
with dask.config.set(num_workers=1):
w.save_datasets([ds2],
sector_id='LCC',
source_name="TESTS",
tile_count=(3, 3),
lettered_grid=True)
all_files = glob(os.path.join(second_base_dir, 'TESTS_AII*.nc'))
# 16 original tiles + 4 new tiles
assert len(all_files) == 20
# these tiles should be the right-most edge of the first image
first_right_edge_files = [
x for x in first_files
if 'P02' in x or 'P04' in x or 'V02' in x or 'V04' in x
]
for new_file in first_right_edge_files:
orig_file = new_file.replace(second_base_dir, first_base_dir)
orig_nc = xr.open_dataset(orig_file)
orig_data = orig_nc['data'].values
if not np.isnan(orig_data).any():
# we only care about the tiles that had NaNs originally
continue
new_nc = xr.open_dataset(new_file)
new_data = new_nc['data'].values
# there should be at least some areas of the file
# that old data was present and hasn't been replaced
np.testing.assert_allclose(orig_data[:, :20], new_data[:, :20])
# it isn't exactly 200 because the tiles aren't aligned with the
# data (the left-most tile doesn't have data until some columns
# in), but it should be at least that many columns
assert np.isnan(orig_data[:, 200:]).all()
assert not np.isnan(new_data[:, 200:]).all()
def test_multivar_numbered_tiles_glm(self, sector):
"""Test creating a tiles with multiple variables."""
import xarray as xr
from satpy.writers.awips_tiled import AWIPSTiledWriter
from xarray import DataArray
from pyresample.geometry import AreaDefinition
from pyresample.utils import proj4_str_to_dict
w = AWIPSTiledWriter(base_dir=self.base_dir, compress=True)
area_def = AreaDefinition(
'test',
'test',
'test',
proj4_str_to_dict(
'+proj=lcc +datum=WGS84 +ellps=WGS84 +lon_0=-95. '
'+lat_0=25 +lat_1=25 +units=m +no_defs'),
100,
200,
(-1000., -1500., 1000., 1500.),
)
now = datetime(2018, 1, 1, 12, 0, 0)
end_time = now + timedelta(minutes=20)
ds1 = DataArray(da.from_array(np.linspace(0.,
1.,
20000,
dtype=np.float32).reshape(
(200, 100)),
chunks=50),
attrs=dict(name='total_energy',
platform_name='GOES-17',
sensor='SENSOR',
units='1',
area=area_def,
start_time=now,
end_time=end_time,
scan_mode='M3',
scene_abbr=sector,
platform_shortname="G17"))
ds2 = ds1.copy()
ds2.attrs.update({
'name': 'flash_extent_density',
})
ds3 = ds1.copy()
ds3.attrs.update({
'name': 'average_flash_area',
})
dqf = ds1.copy()
dqf = (dqf * 255).astype(np.uint8)
dqf.attrs = ds1.attrs.copy()
dqf.attrs.update({
'name': 'DQF',
'_FillValue': 1,
})
w.save_datasets([ds1, ds2, ds3, dqf],
sector_id='TEST',
source_name="TESTS",
tile_count=(3, 3),
template='glm_l2_rad{}'.format(sector.lower()))
all_files = glob(os.path.join(self.base_dir, '*_GLM*.nc'))
assert len(all_files) == 9
for fn in all_files:
ds = xr.open_dataset(fn, mask_and_scale=False)
check_required_common_attributes(ds)
if sector == 'C':
assert ds.attrs['time_coverage_end'] == end_time.strftime(
'%Y-%m-%dT%H:%M:%S.%fZ')
else: # 'F'
assert ds.attrs['time_coverage_end'] == end_time.strftime(
'%Y-%m-%dT%H:%M:%SZ')