def main():
from satpy import Scene
from satpy.writers.scmi import add_backend_argument_groups as add_writer_argument_groups
import argparse
parser = argparse.ArgumentParser(description="Convert GEOCAT Level 1 and 2 to AWIPS SCMI files")
parser.add_argument('-v', '--verbose', dest='verbosity', action="count", default=0,
help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG (default INFO)')
parser.add_argument('-l', '--log', dest="log_fn", default=None,
help="specify the log filename")
subgroups = add_scene_argument_groups(parser)
subgroups += add_writer_argument_groups(parser)
args = parser.parse_args()
scene_args = {ga.dest: getattr(args, ga.dest) for ga in subgroups[0]._group_actions}
load_args = {ga.dest: getattr(args, ga.dest) for ga in subgroups[1]._group_actions}
writer_init_args = {ga.dest: getattr(args, ga.dest) for ga in subgroups[2]._group_actions}
writer_call_args = {ga.dest: getattr(args, ga.dest) for ga in subgroups[3]._group_actions}
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
logging.basicConfig(level=levels[min(3, args.verbosity)], filename=args.log_fn)
scn = Scene(**scene_args)
scn.load(load_args['datasets'])
writer_args = {}
writer_args.update(writer_init_args)
writer_args.update(writer_call_args)
scn.save_datasets(writer='scmi', **writer_args)
def step_impl(context, composite):
from satpy import Scene
scn = Scene(reader=context.dformat,
filenames=get_all_files(os.path.join(context.data_path, 'data'),
'*'))
scn.load([composite])
context.scn = scn
context.composite = composite
def step_impl(context):
from satpy import Scene, find_files_and_readers
from datetime import datetime
os.chdir("/tmp/")
readers_files = find_files_and_readers(sensor='viirs',
start_time=datetime(2015, 3, 11, 11, 20),
end_time=datetime(2015, 3, 11, 11, 26))
scn = Scene(filenames=readers_files)
scn.load(["M02"])
context.scene = scn
def step_impl_user_loads_no_config(context):
from satpy import Scene, find_files_and_readers
from datetime import datetime
os.chdir("/tmp/")
readers_files = find_files_and_readers(
sensor='viirs',
start_time=datetime(2015, 3, 11, 11, 20),
end_time=datetime(2015, 3, 11, 11, 26))
scn = Scene(filenames=readers_files)
scn.load(["M02"])
context.scene = scn
def test_1088(fake_open_dataset):
"""Check that copied arrays gets resampled."""
from satpy import Scene
fake_open_dataset.side_effect = generate_fake_abi_xr_dataset
scene = Scene(abi_file_list, reader='abi_l1b')
scene.load(['C04'], calibration='radiance')
my_id = make_dataid(name='my_name', wavelength=(10, 11, 12))
scene[my_id] = scene['C04'].copy()
resampled = scene.resample('eurol')
assert resampled[my_id].shape == (2048, 2560)
def test_load_250m_cloud_mask_dataset(self, input_files, exp_area):
"""Test loading 250m cloud mask."""
scene = Scene(reader='modis_l2', filenames=input_files)
dataset_name = 'cloud_mask'
scene.load([dataset_name], resolution=250)
cloud_mask_id = make_dataid(name=dataset_name, resolution=250)
assert cloud_mask_id in scene
cloud_mask = scene[cloud_mask_id]
assert cloud_mask.shape == _shape_for_resolution(250)
# mask variables should be integers
assert np.issubdtype(cloud_mask.dtype, np.integer)
assert cloud_mask.attrs.get('_FillValue') is not None
_check_shared_metadata(cloud_mask, expect_area=exp_area)
def calc_historical_rho(datetime, wndw):
'''
This function returns ground and cloud reflectance matrices for the datetime
entered by the user. The first for loop iterates over the time instances,
which is defined by tr_window_length. Although currently fixed, tr_window_length
could be made a function argument.
The function appends all the preceding satellite images and calculates the
5% and 95% for the rho_ground and rho_cloud, respectively.
Arguments:
- period: pandas datetime Timestamp.
- wndw: scalar with the training window length (for validation purposes)
Output:
- Returns numpy arrays of rho_ground and rho_cloud.
'''
dirs = create_historical_filenames(datetime, wndw)
if dirs is None: # Check whether azimuth is lower than 85, skips if not.
return
else:
sat_imgs = []
for filename in dirs: # Loop over the trailing window tr_window_length.
filename = glob(
filename + '*.nat'
) # Because dirs was a list of lists extract first element.
if not filename: # If filename is empty, continue to the next iteration.
continue
else:
global_scene = Scene(reader="seviri_l1b_native",
filenames=filename)
# Load the HRV channel:
global_scene.load(['HRV'])
# Resample:
#local_scene = global_scene.resample("scan1",radius_of_influence=50e3,resampler='nearest',neighbours=16) # nearest='bilinear',cache_dir=REFLECTANCE_DATA_PATH
# radius_of_influence: maximum distance to search for a neighbour for each point in the target grid
#hrv = local_scene['HRV'] # xarray.DataArray
hrv = global_scene['HRV'] # xarray.DataArray
if hrv.values.shape == (
1587, 3840
): # Check only for the original HRV images! Some original HRV images are of different size so check this:
# Add time dimension and coordinate so I can easily slice the resulting xArray:
hrv = hrv.assign_coords(time=hrv.attrs['end_time'])
hrv = hrv.expand_dims('time')
sat_imgs.append(hrv)
# Concatenate the list above on the 'time' dimension
combined = xr.concat(sat_imgs, dim='time')
# Extract the reflectances (this takes a while)
reflectance = combined.values
# Calculate the lowest 5% for each pixel
rho_ground = np.percentile(reflectance, 5, 0)
# Calculate the lowest 5% for each pixel (added 2020-09-14)
rho_cloud = np.percentile(reflectance, 95, 0)
return (rho_ground, rho_cloud)
def main():
from satpy import Scene
from satpy.writers.scmi import add_backend_argument_groups as add_writer_argument_groups
import argparse
parser = argparse.ArgumentParser(
description="Convert GEOCAT Level 1 and 2 to AWIPS SCMI files")
parser.add_argument(
'-v',
'--verbose',
dest='verbosity',
action="count",
default=0,
help=
'each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG (default INFO)'
)
parser.add_argument('-l',
'--log',
dest="log_fn",
default=None,
help="specify the log filename")
subgroups = add_scene_argument_groups(parser)
subgroups += add_writer_argument_groups(parser)
args = parser.parse_args()
scene_args = {
ga.dest: getattr(args, ga.dest)
for ga in subgroups[0]._group_actions
}
load_args = {
ga.dest: getattr(args, ga.dest)
for ga in subgroups[1]._group_actions
}
writer_init_args = {
ga.dest: getattr(args, ga.dest)
for ga in subgroups[2]._group_actions
}
writer_call_args = {
ga.dest: getattr(args, ga.dest)
for ga in subgroups[3]._group_actions
}
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
logging.basicConfig(level=levels[min(3, args.verbosity)],
filename=args.log_fn)
scn = Scene(**scene_args)
scn.load(load_args['datasets'])
writer_args = {}
writer_args.update(writer_init_args)
writer_args.update(writer_call_args)
scn.save_datasets(writer='scmi', **writer_args)
def test_load_l2_dataset(self, input_files, loadables, exp_resolution,
exp_area, exp_value):
"""Load and check an L2 variable."""
scene = Scene(reader='modis_l2', filenames=input_files)
scene.load(loadables)
for ds_name in loadables:
assert ds_name in scene
data_arr = scene[ds_name]
assert isinstance(data_arr.data, da.Array)
data_arr = data_arr.compute()
assert data_arr.values[0, 0] == exp_value
assert data_arr.shape == _shape_for_resolution(exp_resolution)
assert data_arr.attrs.get('resolution') == exp_resolution
_check_shared_metadata(data_arr, expect_area=exp_area)
def preproc_data(infile):
"""Read MODIS data and find good pixels."""
# Load relevant bands
scn = Scene([infile], reader='modis_l1b')
scn.load(['31'], resolution=1000)
# Get lat / lon grid to ensure we focus on tropics
in_lons, in_lats = scn['31'].area.get_lonlats()
in_lats = in_lats.compute()
in_lons = in_lons.compute()
in_lats = in_lats.ravel()
in_lons = in_lons.ravel()
lats2 = np.abs(in_lats)
if np.nanmin(lats2) < 33.5:
scn_data = scn['31'].values
scn_data = scn_data.ravel()
# Only look at points in tropics
gd_pts = (lats2 < 33.5).nonzero()
scn_data = scn_data[gd_pts]
in_lons = in_lons[gd_pts]
in_lats = in_lats[gd_pts]
lats2 = lats2[gd_pts]
# Sometimes there's bad data, <155K BT
gd_pts = (scn_data > 155).nonzero()
scn_data = scn_data[gd_pts]
in_lons = in_lons[gd_pts]
in_lats = in_lats[gd_pts]
lats2 = lats2[gd_pts]
# Now find points colder than 205K for stats
gd_pts = (scn_data < 205).nonzero()
scn_data = scn_data[gd_pts]
in_lons = in_lons[gd_pts]
in_lats = in_lats[gd_pts]
lats2 = lats2[gd_pts]
if len(gd_pts[0]) < 1:
return None, None, None
if np.nanmin(lats2) > 33.5:
return None, None, None
else:
# Every everything is high latitude, return nothing
print(" - All data-points outside latitude range, skipping.")
return None, None, None
# Return data
return in_lats, in_lons, scn_data
def test_load_longitude_latitude(self):
"""Test that longitude and latitude datasets are loaded correctly."""
from satpy import DatasetID
scene = Scene(reader='modis_l2', filenames=[self.file_name])
for dataset_name in ['longitude', 'latitude']:
# Default resolution should be the interpolated 1km
scene.load([dataset_name])
longitude_1km_id = DatasetID(name=dataset_name, resolution=1000)
longitude_1km = scene[longitude_1km_id]
self.assertEqual(longitude_1km.shape, (5*SCAN_WIDTH, 5*SCAN_LEN+4))
# Specify original 5km scale
longitude_5km = scene.load([dataset_name], resolution=5000)
longitude_5km_id = DatasetID(name=dataset_name, resolution=5000)
longitude_5km = scene[longitude_5km_id]
self.assertEqual(longitude_5km.shape, TEST_DATA[dataset_name.capitalize()]['data'].shape)
def test_mcmip_get_dataset(self, xr_):
"""Test getting channel from MCMIP file."""
from datetime import datetime
from pyresample.geometry import AreaDefinition
from satpy import Scene
from satpy.dataset.dataid import WavelengthRange
xr_.open_dataset.return_value = _create_mcmip_dataset()
fn = "OR_ABI-L2-MCMIPF-M6_G16_s20192600241149_e20192600243534_c20192600245360.nc"
scn = Scene(reader='abi_l2_nc', filenames=[fn])
scn.load(['C14'])
exp_data = np.array([[2 * 0.3052037, np.nan],
[32768 * 0.3052037, 32767 * 0.3052037]])
exp_attrs = {
'instrument_ID': None,
'modifiers': (),
'name': 'C14',
'orbital_slot': None,
'reader': 'abi_l2_nc',
'platform_name': 'GOES-16',
'platform_shortname': 'G16',
'production_site': None,
'satellite_altitude': 35786020.,
'satellite_latitude': 0.0,
'satellite_longitude': -89.5,
'scan_mode': 'M6',
'scene_id': None,
'sensor': 'abi',
'timeline_ID': None,
'start_time': datetime(2017, 9, 20, 17, 30, 40, 800000),
'end_time': datetime(2017, 9, 20, 17, 41, 17, 500000),
'calibration': 'brightness_temperature',
'ancillary_variables': [],
'wavelength': WavelengthRange(10.8, 11.2, 11.6, unit='µm'),
'units': 'm'
}
res = scn['C14']
np.testing.assert_allclose(res.data, exp_data, equal_nan=True)
assert isinstance(res.attrs['area'], AreaDefinition)
# don't complicate the comparison below
for key in ('area', '_satpy_id'):
del res.attrs[key]
assert dict(res.attrs) == exp_attrs
def main():
failures = open(r'D:\failures.txt', 'w')
# Get List of downloaded files
file_list = os.listdir(data_dir)
print('Roughly {} data samples found'.format(len(file_list)/3))
for file in file_list:
# Only run loop when you have a geo file
if not file.startswith('VNP03'):
continue
# Get the band file
if os.path.exists(os.path.join(pp_dir, file[:-3] + '.png')):
continue
geo_path = os.path.join(data_dir, file)
subs = file[:4] + '2' + file[5:24]
band_path = [i for i in file_list if subs in i]
if len(band_path) != 1:
print('failure {} {}'.format(file, band_path))
break
band_path = os.path.join(data_dir, band_path[0])
# Get comparable GOES file
v_time = datetime.strptime(file[10:22], '%Y%j.%H%M')
g_subs = 'C07_G17_s{}'.format(file[10:17])
g_files = [i for i in file_list if g_subs in i]
goes_file = None
for g_file in g_files:
g_time = datetime.strptime(g_file[27:38], '%Y%j%H%M')
tm_delta = v_time - g_time
if abs(tm_delta.total_seconds()) < 4*60:
goes_file = g_file
if not goes_file:
print('No Goes File for {}'.format(file))
failures.write("No Match found for {}\n".format(file))
continue
# Load SatPy Scenes
viirs_files = [band_path, geo_path]
goes_files = [os.path.join(data_dir, goes_file)]
viirs_scene = Scene(reader = v_reader, filenames = viirs_files)
goes_scene = Scene(reader = g_reader, filenames = goes_files)
viirs_scene.load(['I04'])
goes_scene.load(['C07'])
# Resample and Save PNGs
print(file)
rs = viirs_scene.resample(viirs_scene['I04'].attrs['area'], resampler = 'nearest')
rs.save_dataset('I04', os.path.join(pp_dir, file[:-3] + '.png'))
rs_g = goes_scene.resample(viirs_scene['I04'].attrs['area'], resampler = 'nearest')
rs_g.save_dataset('C07', os.path.join(pp_dir, goes_file[:-3] + '.png'))
failures.close
def _get_fake_g16_abi_c01_scene(mocker):
attrs = {
'name': 'C01',
'wavelength': (1.0, 2.0, 3.0),
'area': AreaDefinition(
'test', 'test', 'test',
{
'proj': 'geos',
'sweep': 'x',
'lon_0': -75,
'h': 35786023,
'ellps': 'GRS80',
'units': 'm',
}, 5, 5,
(-5434894.885056, -5434894.885056, 5434894.885056, 5434894.885056)
),
'start_time': datetime(2018, 9, 10, 17, 0, 31, 100000),
'end_time': datetime(2018, 9, 10, 17, 11, 7, 800000),
'standard_name': 'toa_bidirectional_reflectance',
'sensor': 'abi',
'platform_name': 'GOES-16',
'platform_shortname': 'G16',
}
data_arr = xr.DataArray(da.from_array(np.empty((5, 5), dtype=np.float64), chunks='auto'),
attrs=attrs)
scn = Scene()
scn['C01'] = data_arr
scn.load = mocker.MagicMock() # don't do anything on load
return scn
def read_sat_data(fname, channels, reader):
"""Read satellite data"""
if not isinstance(fname, (list, set, tuple)):
fname = [fname, ]
glbl = Scene(filenames=fname, reader=reader)
glbl.load(channels)
# Compute the dask arrays
for chan in channels:
logging.info("Loading %s", chan)
try:
glbl[chan] = glbl[chan].compute()
except KeyError:
logging.error("Channel %s not available", chan)
return None
glbl.attrs["proc_time"] = dt.datetime.utcnow()
return glbl
def test_orbital_parameters(self):
"""Test that the orbital parameters in attributes are handled correctly."""
filename = 'testingcfwriter4{:s}-viirs-mband-20201007075915-20201007080744.nc'.format(
datetime.utcnow().strftime('%Y%j%H%M%S'))
try:
self.scene.save_datasets(writer='cf',
filename=filename,
header_attrs={'instrument': 'avhrr'})
scn_ = Scene(reader='satpy_cf_nc', filenames=[filename])
scn_.load(['image0'])
orig_attrs = self.scene['image0'].attrs['orbital_parameters']
new_attrs = scn_['image0'].attrs['orbital_parameters']
assert isinstance(new_attrs, dict)
for key in orig_attrs:
assert orig_attrs[key] == new_attrs[key]
finally:
with suppress(PermissionError):
os.remove(filename)
def process_set(grouped_files, curr_idx, total_groups):
"""process_ALLis a list of parsed filenames (DNB, Mband, ABI, Cband)
Given these files, use Scene to load the appropriate channels.
Then resample (colocate) to make the channels match up.
Then save these colocated channels.
Crop the NaN edges, tag with meta information (which files were used as input),
And finally save the numpy arrays (so we don't need to recompute next time)"""
log.info(
f'{rgb(255,0,0)}Processing{reset} timestep {bold}{curr_idx + 1}/{total_groups}{reset}'
)
dt = grouped_files['viirs'][0]["datetime"]
viirsfiles = [f["path"] for f in grouped_files['viirs']]
abifiles = [f["path"] for f in grouped_files['abi']]
master_scene = Scene(filenames={
'viirs_sdr': viirsfiles,
'abi_l1b': abifiles
})
master_scene.load(VIIRS_channels + ABI_channels + lat_long_both)
#load and pair the reflectance
reflectfile = grouped_files['reflectance']['path']
Reflectance = xarray.open_dataset(reflectfile)
swath_def = SwathDefinition(Reflectance['longitude'],
Reflectance['latitude'])
sm_refl = Reflectance['SM_Reflectance']
sm_refl.attrs['area'] = swath_def
#bring reflectance back to the satpy "Scene"
master_scene['SM_Reflectance'] = sm_refl
resample_scn = master_scene.resample(master_scene['DNB'].attrs['area'],
resampler='nearest')
log.info(f'Cropping nan edges of {blue}{dt}{reset}')
t = time.time()
data = crop.crop_nan_edges(resample_scn, crop_channels, all_channels)
log.debug(
f'Cropping nan edges took {rgb(255,0,0)}{time.time() - t:.2f}{reset} seconds'
)
data['channels'] = list(data)
data['filenames'] = viirsfiles + abifiles + [reflectfile]
data["datetime"] = dt
return data
def read_image(filepath):
"""Read the image from *filepath* and return it as PIL Image object."""
scn = Scene(reader='generic_image', filenames=[
filepath,
])
scn.load(['image'])
y_size = scn['image']['y'].size
x_size = scn['image']['x'].size
arr = np.array(scn['image'])
alpha = np.isnan(arr[0, :, :])
arr = arr.astype(np.uint8)
if np.any(alpha):
alpha = 255 * np.invert(alpha).astype(np.uint8)
alpha = alpha.reshape(1, y_size, x_size)
arr = np.vstack((arr, alpha))
arr = np.squeeze(np.rollaxis(arr, 0, 3))
img = Image.fromarray(arr)
return img
def get_cloud_mask(l1_filename, cloud_mask_dir):
""" return a 2d mask, with cloudy pixels marked as 1, non-cloudy pixels marked as 0 """
basename = os.path.split(l1_filename)
cloud_mask_filename = glob.glob(
os.path.join(cloud_mask_dir,
'MYD35*' + l1_filename.split('.A')[1][:12] + '*'))[0]
# satpy returns(0=Cloudy, 1=Uncertain, 2=Probably Clear, 3=Confident Clear)
swath = Scene(reader='modis_l2', filenames=[cloud_mask_filename])
swath.load(['cloud_mask'], resolution=1000)
cloud_mask = np.array(swath['cloud_mask'].load())[:MAX_HEIGHT, :MAX_WIDTH]
cloud_mask = (cloud_mask == 0)
cloud_mask = cloud_mask.astype(np.intp)
return cloud_mask
def test_load_category_dataset(self, input_files, loadables,
request_resolution, exp_resolution,
exp_area):
"""Test loading category products."""
scene = Scene(reader='modis_l2', filenames=input_files)
kwargs = {
"resolution": request_resolution
} if request_resolution is not None else {}
scene.load(loadables, **kwargs)
for ds_name in loadables:
cat_id = make_dataid(name=ds_name, resolution=exp_resolution)
assert cat_id in scene
cat_data_arr = scene[cat_id]
assert cat_data_arr.shape == _shape_for_resolution(exp_resolution)
assert cat_data_arr.attrs.get('resolution') == exp_resolution
# mask variables should be integers
assert np.issubdtype(cat_data_arr.dtype, np.integer)
assert cat_data_arr.attrs.get('_FillValue') is not None
_check_shared_metadata(cat_data_arr, expect_area=exp_area)
def get_swath(radiance_filename, myd03_dir):
"""
:param radiance_filename: MYD02 filename
:param myd03_dir: root directory of MYD03 geolocational files
:return swath: numpy.ndarray of size (15, HEIGHT, WIDTH)
Uses the satpy Scene reader with the modis-l1b files. Issues reading files might be due to pyhdf not being
installed - otherwise try pip install satpy[modis_0l1b]
Creates a scene with the MYD02 and MYD03 files, and extracts them as multi-channel arrays. The lat and long are
are appended as additional channels.
"""
# bands selected from MODIS
composite = [
'1', '2', '29', '33', '34', '35', '36', '26', '27', '20', '21', '22',
'23'
]
# find a corresponding geolocational (MYD03) file for the provided radiance (MYD02) file
geoloc_filename = find_matching_geoloc_file(radiance_filename, myd03_dir)
# load the global scene using satpy
global_scene = Scene(reader='modis_l1b',
filenames=[radiance_filename, geoloc_filename])
# load composite, resolution of 1km
global_scene.load(composite, resolution=1000)
# load latitudes and longitudes, resolution 1km
global_scene.load(['latitude', 'longitude'], resolution=1000)
latitude = np.array(global_scene['latitude'].load())
longitude = np.array(global_scene['longitude'].load())
swath = []
for comp in composite:
temp = np.array(global_scene[comp].load())
swath.append(temp[:MAX_HEIGHT, :MAX_WIDTH])
swath.append(latitude[:MAX_HEIGHT, :MAX_WIDTH])
swath.append(longitude[:MAX_HEIGHT, :MAX_WIDTH])
return np.array(swath, dtype=np.float16)
def test_scene_dataset_values(self):
"""Test loading data."""
from satpy import Scene
fname = os.path.join(self.base_dir, FILENAME)
scn = Scene(reader='ascat_l2_soilmoisture_bufr', filenames=[fname])
for name in scn.available_dataset_names():
scn.load([name])
loaded_values = scn[name].values
fill_value = scn[name].attrs['fill_value']
# replace nans in data loaded from file with the fill value defined in the .yaml
# to make them comparable
loaded_values_nan_filled = np.nan_to_num(loaded_values,
nan=fill_value)
key = scn[name].attrs['key']
original_values = MSG[key]
# this makes each assertion below a separate test from unittest's point of view
# (note: if all subtests pass, they will count as one test)
with self.subTest(msg="Test failed for dataset: " + name):
self.assertTrue(
np.allclose(original_values, loaded_values_nan_filled))
def test_scene_loading(self, calibrate, *mocks):
"""Test masking of space pixels."""
from satpy import Scene
nrows = 25
ncols = 100
calibrate.return_value = np.ones((nrows, ncols))
with _fake_hsd_handler() as fh:
with mock.patch('satpy.readers.ahi_hsd.AHIHSDFileHandler') as fh_cls:
fh_cls.return_value = fh
fh.filename_info['total_segments'] = 1
fh.filename_info['segment'] = 1
fh.data_info['number_of_columns'] = ncols
fh.data_info['number_of_lines'] = nrows
scn = Scene(reader='ahi_hsd', filenames=['HS_H08_20210225_0700_B07_FLDK_R20_S0110.DAT'])
scn.load(['B07'])
im = scn['B07']
# Make sure space masking worked
mask = im.to_masked_array().mask
ref_mask = np.logical_not(get_geostationary_mask(fh.area).compute())
np.testing.assert_equal(mask, ref_mask)
def read(self, filename, fields=None, **kwargs):
scene = Scene(reader=self.satpy_reader, filenames=[filename.path])
# If the user has not passed any fields to us, we load all per default.
if fields is None:
fields = scene.available_dataset_ids()
# Load all selected fields
scene.load(fields, **kwargs)
if isinstance(fields[0], str):
data_arrays = {field: scene.get(field) for field in fields}
else:
data_arrays = {field.name: scene.get(field) for field in fields}
for name, array in data_arrays.items():
array.name = name
dataset = xr.merge(data_arrays.values())
return dataset
def test_load_chlor_a(self, input_files, exp_plat, exp_sensor, exp_rps,
apply_quality_flags):
"""Test that we can load 'chlor_a'."""
reader_kwargs = {"apply_quality_flags": apply_quality_flags}
scene = Scene(reader='seadas_l2',
filenames=input_files,
reader_kwargs=reader_kwargs)
scene.load(['chlor_a'])
data_arr = scene['chlor_a']
assert data_arr.attrs['platform_name'] == exp_plat
assert data_arr.attrs['sensor'] == exp_sensor
assert data_arr.attrs['units'] == 'mg m^-3'
assert data_arr.dtype.type == np.float32
assert isinstance(data_arr.attrs["area"], SwathDefinition)
assert data_arr.attrs["rows_per_scan"] == exp_rps
data = data_arr.data.compute()
if apply_quality_flags:
assert np.isnan(data[2, 2])
assert np.count_nonzero(np.isnan(data)) == 1
else:
assert np.count_nonzero(np.isnan(data)) == 0
def simple_export(hrit_files, time, dpi, photo_path, load_photo=['VIS006']):
from satpy import Scene
from satpy import find_files_and_readers
from datetime import datetime
import matplotlib as mpl
mpl.rcParams['figure.dpi'] = dpi
# load_photo = 'VIS006'
first = 0
last = len(time) - 1
# print(len(time),last)
yearF, monthF, dayF, hourF, minuteF, secondF = time[first].tt_calendar()
yearL, monthL, dayL, hourL, minuteL, secondL = time[last].tt_calendar()
# IT IS NOT WORKING FIND SOLUTION - Adding a minute in case there is only one point on map
if len(time) == 1:
time[last].tt = time[last].tt + 1 / 3600
yearL, monthL, dayL, hourL, minuteL, secondL = time[last].tt_calendar()
# print("It works")
# print(yearF, monthF, dayF, hourF, minuteF, secondF )
# print(yearL, monthL, dayL, hourL, minuteL, secondL)
# time[0].tt_calendar()[0]
files = find_files_and_readers(base_dir=hrit_files,
start_time=datetime(yearF, monthF, dayF,
hourF, minuteF),
end_time=datetime(yearL, monthL, dayL,
hourL, minuteL),
reader='seviri_l1b_hrit')
scn = Scene(filenames=files)
scn.load(load_photo)
file = photo_path + 'globe_' + load_photo[
0] + '_{date:%Y-%m-%d_%H_%M_%S}.png'.format(date=scn.start_time)
scn.save_dataset(load_photo[0],
writer='simple_image',
filename=file,
num_threads=8)
return ()
def load_sat_polar(in_reader, globber, chans, latpos, lonpos, deller):
"""Load data for a given sensor, find BT at particular lat/lon."""
storm_bbox = [
lonpos - deller, latpos - deller, lonpos + deller, latpos + deller
]
files = glob(globber)
if len(files) < 1:
return None
scn = Scene(files, reader=in_reader)
scn.load(chans)
lons = None
lats = None
output_dict = {}
for chan in chans:
try:
if lons is None or lats is None:
lons, lats = scn[chan].attrs['area'].get_lonlats()
try:
lats = lats.compute()
lons = lons.compute()
except:
pass
btdata = np.array(scn[chan])
pts = (lons < storm_bbox[0]).nonzero()
btdata[pts] = np.nan
pts = (lats < storm_bbox[1]).nonzero()
btdata[pts] = np.nan
pts = (lons > storm_bbox[2]).nonzero()
btdata[pts] = np.nan
pts = (lats > storm_bbox[3]).nonzero()
btdata[pts] = np.nan
output_dict[chan] = [
np.nanmin(btdata), scn[chan].wavelength.min,
scn[chan].wavelength.central, scn[chan].wavelength.max
]
except KeyError:
output_dict[chan] = [None, None, None, None]
return output_dict
def load_himawari(indir, in_time, comp_type, timedelt, mode):
"""
Load a Himawari/AHI scene as given by img_time.
img_time should be the *start* time for the scan, as the ending time
will be automatically defined from this using timedelt
The image will be loaded with Satpy, return value is a cartopy object
Arguments:
indir - directory holding the Himawari data in HSD (unzipped) format
img_time - a datetime indicating the scene start time in UTC
comp_type - the Satpy composite to create (true_color, B03, etc)
timedelt - the scanning time delta (10 min for full disk AHI)
mode - scanning mode (FD = Full disk, MESO = Mesoscale sector)
Returns:
sat_data - the satellite data object, unresampled
"""
if mode == 'MESO':
tmp_t = in_time
minu = tmp_t.minute
minu = minu - (minu % 10)
tmp_t = tmp_t.replace(minute=minu)
tmp_t = tmp_t.replace(second=0)
dt = (in_time - tmp_t).total_seconds() / 60.
src_str = '*_R30' + str(int(dt/timedelt) + 1) + '*'
dtstr = tmp_t.strftime("%Y%m%d_%H%M")
files = glob(indir + '*' + dtstr + src_str + '.DAT')
files.sort()
else:
files = ffar(start_time=in_time,
end_time=in_time + timedelta(minutes=timedelt-1),
base_dir=indir,
reader='ahi_hsd')
scn = Scene(reader='ahi_hsd', filenames=files)
scn.load([comp_type], pad_data=False)
return scn
def process_trio(trio, curr_idx, len_trio):
"""trio is a list of three parsed filenames (see the function parse_filename below).
Given these three files, use Scene to load the appropriate channels.
Then resample (colocate) to make the channels match up.
Then save these colocated channels.
Crop the NaN edges, tag with meta information (which files were used as input),
And finally save the numpy arrays (so we don't need to recompute next time)"""
dt = trio[0]["datetime"]
log.info(
f'{rgb(255,0,0)}Processing{reset} timestep {bold}{curr_idx + 1}/{len_trio}{reset} {blue}{dt}{reset} '
)
#load the sat data
scn = Scene(
reader='viirs_sdr',
filenames=[f['path'] for f in trio if f['filename'].endswith(".h5")])
scn.load(viirs_channels + lat_long_both)
#load and pair the reflectance
Reflectance = xarray.open_dataset(find_ncfile(trio)['path'])
swath_def = SwathDefinition(Reflectance['longitude'],
Reflectance['latitude'])
sm_refl = Reflectance['SM_Reflectance']
sm_refl.attrs['area'] = swath_def
#bring reflectance back to the satpy "Scene"
scn['SM_reflectance'] = sm_refl
log.info(f'Resampling {blue}{dt}{reset}')
resample_scn = scn.resample(scn['DNB'].attrs['area'], resampler='nearest')
log.info(f'Cropping nan edges of {blue}{dt}{reset}')
t = time.time()
data = crop.crop_nan_edges(resample_scn, all_channels)
log.debug(
f'Cropping nan edges took {rgb(255,0,0)}{time.time() - t:.2f}{reset} seconds'
)
data['channels'] = list(data)
data['filenames'] = [f['filename'] for f in trio]
data["datetime"] = dt
return data
def test_satpy_importer_contour_0_360(tmpdir, monkeypatch, mocker):
"""Test import of grib contour data using Satpy."""
db_sess = mocker.MagicMock()
attrs = {
'name': 'gh',
'level': 125,
'area': AreaDefinition(
'test', 'test', 'test',
{
'proj': 'eqc',
'lon_0': 0,
'pm': 180,
'R': 6371229,
}, 240, 120,
(-20015806.220738243, -10007903.110369122, 20015806.220738243, 10007903.110369122)
),
'start_time': datetime(2018, 9, 10, 17, 0, 31, 100000),
'end_time': datetime(2018, 9, 10, 17, 11, 7, 800000),
'model_time': datetime(2018, 9, 10, 17, 11, 7, 800000),
'standard_name': 'geopotential_height',
}
data_arr = xr.DataArray(da.from_array(np.random.random((120, 240)).astype(np.float64), chunks='auto'),
attrs=attrs)
scn = Scene()
scn['gh'] = data_arr
scn.load = mocker.MagicMock() # don't do anything on load
imp = SatpyImporter(['/test/file.nc'], tmpdir, db_sess,
scene=scn,
reader='grib',
dataset_ids=[make_dataid(name='gh', level=125)])
imp.merge_resources()
assert imp.num_products == 1
products = list(imp.merge_products())
assert len(products) == 1
assert products[0].info[Info.STANDARD_NAME] == 'geopotential_height'
assert products[0].info[Info.KIND] == Kind.CONTOUR
query_mock = mocker.MagicMock(name='query')
filter1_mock = mocker.MagicMock(name='filter1')
filter2_mock = mocker.MagicMock(name='filter2')
db_sess.query.return_value = query_mock
query_mock.filter.return_value = filter1_mock
filter1_mock.filter.return_value = filter2_mock
filter2_mock.all.return_value = products
import_gen = imp.begin_import_products()
content_progresses = list(import_gen)
# image and contour content
assert len(content_progresses) == 2
# make sure data was swapped to -180/180 space
assert (content_progresses[0].data[:, :120] == data_arr.data[:, 120:].astype(np.float32)).all()
assert (content_progresses[0].data[:, 120:] == data_arr.data[:, :120].astype(np.float32)).all()
def test_load_vis_saturation(self, mask_saturated, modis_l1b_nasa_mod021km_file):
"""Test loading visible band."""
scene = Scene(reader='modis_l1b', filenames=modis_l1b_nasa_mod021km_file,
reader_kwargs={"mask_saturated": mask_saturated})
dataset_name = '2'
scene.load([dataset_name])
dataset = scene[dataset_name]
assert dataset.shape == _shape_for_resolution(1000)
assert dataset.attrs['resolution'] == 1000
_check_shared_metadata(dataset)
# check saturation fill values
data = dataset.values
assert np.isnan(data[-1, -1]) # normal fill value
if mask_saturated:
assert np.isnan(data[-1, -2]) # saturation
assert np.isnan(data[-1, -3]) # can't aggregate
else:
# test data factor/offset are 1/0
# albedos are converted to %
assert data[-1, -2] >= 32767 * 100.0 # saturation
assert data[-1, -3] >= 32767 * 100.0 # can't aggregate
def get_swath_rgb(radiance_filename, myd03_dir, composite='true_color'):
"""
:param radiance_filename: MYD02 filename
:param myd03_dir: root directory of MYD03 geolocational files
:return visible RGB channels: numpy.ndarray of size (3, 2030, 1354)
Uses the satpy Scene reader with the modis-l1b files. Issues reading files might be due to pyhdf not being
installed - otherwise try pip install satpy[modis_0l1b]
Creates a scene with the MYD02 file, and extracts the RGB channels from the 1, 4, 3 visible MODIS bands.
"""
# find a corresponding geolocational (MOD03) file for the provided radiance (MYD02) file
geoloc_filename = find_matching_geoloc_file(radiance_filename, myd03_dir)
global_scene = Scene(reader='modis_l1b',
filenames=[radiance_filename, geoloc_filename])
# load it in, make sure resolution is 1000 to match our other datasets
global_scene.load([composite], resolution=1000)
rgb = np.array(global_scene[composite])[:, :MAX_HEIGHT, :MAX_WIDTH]
return rgb
def importCM(data, myd03_p, cor_inds):
tnames = []
m03_hdf = SD(myd03_p, SDC.READ)
d = m03_hdf.Input_Files
sts = [i + 6 for i in range(len(d)) if d.startswith('MYD03.A', i)]
eds = [i - 18 for i in range(len(d)) if d.startswith('.hdf', i)]
eds.pop(0)
for i in range(len(sts)):
tnames.append(d[sts[i]:eds[i]])
ifi = m03_hdf.select('Input_File_Index')
ifp = m03_hdf.select('Input_Pixel_Index')
ifi = ifi[:]
ifp = ifp[:, :]
print(ifp[0], ifp[1][0])
cm_results = [-9999 for i in range(len(data))]
current_file = ''
files_used = []
for index, d in enumerate(data):
if not d == -9999:
if not tnames[ifi[index]] in current_file:
current_file = myd35_dict.get(tnames[ifi[index]])
print(current_file)
files_used.append(current_file)
swath = Scene(reader='modis_l2',
filenames=[myd35_path + current_file])
swath.load(['cloud_mask'], resolution=1000)
cm = swath['cloud_mask'].values
cm_results[index] = cm[ifp[index][0]][ifp[index][1]] # pylint: disable=E1136
#cm_results.append()
# print(len(cm_results))
# print(len(ifi), len(ifp))
fillVal(ifi, cor_inds, len(ifi))
fillVal(ifp, cor_inds, len(ifp))
# print(len(ifi), len(ifp))
#print(tnames)
return cm_results, files_used
hourS = "%02d" % hour
minS = "%02d" % min
filenames = glob("/var/tmp/cll/data/H-*MSG4*"+yearS+monthS+dayS+hourS+minS+"*__")
global_scene = Scene(reader="hrit_msg", filenames=filenames)
# first try, it stays here only for the memory
# global_scene.load(["HRV", "IR_108"])
# local_scene = global_scene.resample("ccs4")
# lonlats = local_scene["HRV"].area.get_lonlats()
# sza = sun_zenith_angle(local_scene.start_time, lonlats[0], lonlats[1])
# ds = DataArray(sza, dims=['y','x'])
# local_scene['sza'] = ds
# end of the first try, stuff below here is working again
global_scene.load(["ir108", "hrv", "IR_108", "hrv_with_ir"])
local_scene = global_scene.resample("ccs4")
local_scene.load(["hrv_with_ir", "IR_108"])
swiss = load_area("/opt/users/cll/cllwork/etc_work/areas.def", "ccs4")
tmpFileA = "/tmp/welcome.png"
tmpFileB = "/tmp/welcome-ir.png"
outputFile = "/var/tmp/cll/out/PY_visir-ch_"+yearS+monthS+dayS+hourS+minS+".png"
bgFile = "/opt/users/cll/cllwork/ccs4.png"
local_scene.save_dataset("hrv_with_ir", tmpFileA)
local_scene.save_dataset("ir108", tmpFileB)
background = Image.open(bgFile)
foreground = Image.open(tmpFileA)
background = background.convert("RGBA")
foreground.putalpha(foreground.convert('L'))
def main(argv=sys.argv[1:]):
global LOG
from satpy import Scene
from satpy.resample import get_area_def
from satpy.writers import compute_writer_results
from dask.diagnostics import ProgressBar
from polar2grid.core.script_utils import (
setup_logging, rename_log_file, create_exc_handler)
import argparse
prog = os.getenv('PROG_NAME', sys.argv[0])
# "usage: " will be printed at the top of this:
usage = """
%(prog)s -h
see available products:
%(prog)s -r <reader> -w <writer> --list-products -f file1 [file2 ...]
basic processing:
%(prog)s -r <reader> -w <writer> [options] -f file1 [file2 ...]
basic processing with limited products:
%(prog)s -r <reader> -w <writer> [options] -p prod1 prod2 -f file1 [file2 ...]
"""
parser = argparse.ArgumentParser(prog=prog, usage=usage,
description="Load, composite, resample, and save datasets.")
parser.add_argument('-v', '--verbose', dest='verbosity', action="count", default=0,
help='each occurrence increases verbosity 1 level through ERROR-WARNING-INFO-DEBUG (default INFO)')
parser.add_argument('-l', '--log', dest="log_fn", default=None,
help="specify the log filename")
parser.add_argument('--progress', action='store_true',
help="show processing progress bar (not recommended for logged output)")
parser.add_argument('--num-workers', type=int, default=4,
help="specify number of worker threads to use (default: 4)")
parser.add_argument('--match-resolution', dest='preserve_resolution', action='store_false',
help="When using the 'native' resampler for composites, don't save data "
"at its native resolution, use the resolution used to create the "
"composite.")
parser.add_argument('-w', '--writers', nargs='+',
help='writers to save datasets with')
parser.add_argument("--list-products", dest="list_products", action="store_true",
help="List available reader products and exit")
subgroups = add_scene_argument_groups(parser)
subgroups += add_resample_argument_groups(parser)
argv_without_help = [x for x in argv if x not in ["-h", "--help"]]
args, remaining_args = parser.parse_known_args(argv_without_help)
# get the logger if we know the readers and writers that will be used
if args.reader is not None and args.writers is not None:
glue_name = args.reader + "_" + "-".join(args.writers or [])
LOG = logging.getLogger(glue_name)
# add writer arguments
if args.writers is not None:
for writer in (args.writers or []):
parser_func = WRITER_PARSER_FUNCTIONS.get(writer)
if parser_func is None:
continue
subgroups += parser_func(parser)
args = parser.parse_args(argv)
if args.reader is None:
parser.print_usage()
parser.exit(1, "\nERROR: Reader must be provided (-r flag).\n"
"Supported readers:\n\t{}\n".format('\n\t'.join(['abi_l1b', 'ahi_hsd', 'hrit_ahi'])))
if args.writers is None:
parser.print_usage()
parser.exit(1, "\nERROR: Writer must be provided (-w flag) with one or more writer.\n"
"Supported writers:\n\t{}\n".format('\n\t'.join(['geotiff'])))
def _args_to_dict(group_actions):
return {ga.dest: getattr(args, ga.dest) for ga in group_actions if hasattr(args, ga.dest)}
scene_args = _args_to_dict(subgroups[0]._group_actions)
load_args = _args_to_dict(subgroups[1]._group_actions)
resample_args = _args_to_dict(subgroups[2]._group_actions)
writer_args = {}
for idx, writer in enumerate(args.writers):
sgrp1, sgrp2 = subgroups[3 + idx * 2: 5 + idx * 2]
wargs = _args_to_dict(sgrp1._group_actions)
if sgrp2 is not None:
wargs.update(_args_to_dict(sgrp2._group_actions))
writer_args[writer] = wargs
# get default output filename
if 'filename' in wargs and wargs['filename'] is None:
wargs['filename'] = get_default_output_filename(args.reader, writer)
if not args.filenames:
parser.print_usage()
parser.exit(1, "\nERROR: No data files provided (-f flag)\n")
# Prepare logging
rename_log = False
if args.log_fn is None:
rename_log = True
args.log_fn = glue_name + "_fail.log"
levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
setup_logging(console_level=levels[min(3, args.verbosity)], log_filename=args.log_fn)
logging.getLogger('rasterio').setLevel(levels[min(2, args.verbosity)])
sys.excepthook = create_exc_handler(LOG.name)
if levels[min(3, args.verbosity)] > logging.DEBUG:
import warnings
warnings.filterwarnings("ignore")
LOG.debug("Starting script with arguments: %s", " ".join(sys.argv))
# Set up dask and the number of workers
if args.num_workers:
from multiprocessing.pool import ThreadPool
dask.config.set(pool=ThreadPool(args.num_workers))
# Parse provided files and search for files if provided directories
scene_args['filenames'] = get_input_files(scene_args['filenames'])
# Create a Scene, analyze the provided files
LOG.info("Sorting and reading input files...")
try:
scn = Scene(**scene_args)
except ValueError as e:
LOG.error("{} | Enable debug message (-vvv) or see log file for details.".format(str(e)))
LOG.debug("Further error information: ", exc_info=True)
return -1
except OSError:
LOG.error("Could not open files. Enable debug message (-vvv) or see log file for details.")
LOG.debug("Further error information: ", exc_info=True)
return -1
if args.list_products:
print("\n".join(sorted(scn.available_dataset_names(composites=True))))
return 0
# Rename the log file
if rename_log:
rename_log_file(glue_name + scn.attrs['start_time'].strftime("_%Y%m%d_%H%M%S.log"))
# Load the actual data arrays and metadata (lazy loaded as dask arrays)
if load_args['products'] is None:
try:
reader_mod = importlib.import_module('polar2grid.readers.' + scene_args['reader'])
load_args['products'] = reader_mod.DEFAULT_PRODUCTS
LOG.info("Using default product list: {}".format(load_args['products']))
except (ImportError, AttributeError):
LOG.error("No default products list set, please specify with `--products`.")
return -1
LOG.info("Loading product metadata from files...")
scn.load(load_args['products'])
resample_kwargs = resample_args.copy()
areas_to_resample = resample_kwargs.pop('grids')
grid_configs = resample_kwargs.pop('grid_configs')
resampler = resample_kwargs.pop('resampler')
if areas_to_resample is None and resampler in [None, 'native']:
# no areas specified
areas_to_resample = ['MAX']
elif areas_to_resample is None:
raise ValueError("Resampling method specified (--method) without any destination grid/area (-g flag).")
elif not areas_to_resample:
# they don't want any resampling (they used '-g' with no args)
areas_to_resample = [None]
has_custom_grid = any(g not in ['MIN', 'MAX', None] for g in areas_to_resample)
if has_custom_grid and resampler == 'native':
LOG.error("Resampling method 'native' can only be used with 'MIN' or 'MAX' grids "
"(use 'nearest' method instead).")
return -1
p2g_grid_configs = [x for x in grid_configs if x.endswith('.conf')]
pyresample_area_configs = [x for x in grid_configs if not x.endswith('.conf')]
if not grid_configs or p2g_grid_configs:
# if we were given p2g grid configs or we weren't given any to choose from
from polar2grid.grids import GridManager
grid_manager = GridManager(*p2g_grid_configs)
else:
grid_manager = {}
if pyresample_area_configs:
from pyresample.utils import parse_area_file
custom_areas = parse_area_file(pyresample_area_configs)
custom_areas = {x.area_id: x for x in custom_areas}
else:
custom_areas = {}
ll_bbox = resample_kwargs.pop('ll_bbox')
if ll_bbox:
scn = scn.crop(ll_bbox=ll_bbox)
wishlist = scn.wishlist.copy()
preserve_resolution = get_preserve_resolution(args, resampler, areas_to_resample)
if preserve_resolution:
preserved_products = set(wishlist) & set(scn.datasets.keys())
resampled_products = set(wishlist) - preserved_products
# original native scene
to_save = write_scene(scn, args.writers, writer_args, preserved_products)
else:
preserved_products = set()
resampled_products = set(wishlist)
to_save = []
LOG.debug("Products to preserve resolution for: {}".format(preserved_products))
LOG.debug("Products to use new resolution for: {}".format(resampled_products))
for area_name in areas_to_resample:
if area_name is None:
# no resampling
area_def = None
elif area_name == 'MAX':
area_def = scn.max_area()
elif area_name == 'MIN':
area_def = scn.min_area()
elif area_name in custom_areas:
area_def = custom_areas[area_name]
elif area_name in grid_manager:
from pyresample.geometry import DynamicAreaDefinition
p2g_def = grid_manager[area_name]
area_def = p2g_def.to_satpy_area()
if isinstance(area_def, DynamicAreaDefinition) and p2g_def['cell_width'] is not None:
area_def = area_def.freeze(scn.max_area(),
resolution=(abs(p2g_def['cell_width']), abs(p2g_def['cell_height'])))
else:
area_def = get_area_def(area_name)
if resampler is None and area_def is not None:
rs = 'native' if area_name in ['MIN', 'MAX'] else 'nearest'
LOG.debug("Setting default resampling to '{}' for grid '{}'".format(rs, area_name))
else:
rs = resampler
if area_def is not None:
LOG.info("Resampling data to '%s'", area_name)
new_scn = scn.resample(area_def, resampler=rs, **resample_kwargs)
elif not preserve_resolution:
# the user didn't want to resample to any areas
# the user also requested that we don't preserve resolution
# which means we have to save this Scene's datasets
# because they won't be saved
new_scn = scn
to_save = write_scene(new_scn, args.writers, writer_args, resampled_products, to_save=to_save)
if args.progress:
pbar = ProgressBar()
pbar.register()
LOG.info("Computing products and saving data to writers...")
compute_writer_results(to_save)
LOG.info("SUCCESS")
return 0
parser.add_argument('--sat_id', dest='sat_id', action="store", help="Satellite ID", default="8888")
parser.add_argument('--data_cat', dest='data_cat', action="store", help="Category of data (one of GORN, GPRN, P**N)", default="GORN")
parser.add_argument('--area', dest='areadef', action="store", help="Area name, the definition must exist in your areas configuration file", default="nrEURO1km_NPOL_COALeqc")
parser.add_argument('--ph_unit', dest='ph_unit', action="store", help="Physical unit", default="CELSIUS")
parser.add_argument('--data_src', dest='data_src', action="store", help="Data source", default="EUMETCAST")
args = parser.parse_args()
if (args.input_dir != None):
os.chdir(args.input_dir)
cfg = vars(args)
if (args.cfg != None):
with open(args.cfg, 'r') as ymlfile:
cfg = yaml.load(ymlfile)
narea = get_area_def(args.areadef)
global_data = Scene(sensor="images", reader="generic_image", area=narea)
global_data.load(['image'])
global_data['image'].info['area'] = narea
fname = global_data['image'].info['filename']
ofname = fname[:-3] + "tif"
#global_data.save_dataset('image', filename="out.png", writer="simple_image")
global_data.save_dataset('image', filename=ofname, writer="ninjotiff",
sat_id=cfg['sat_id'],
chan_id=cfg['chan_id'],
data_cat=cfg['data_cat'],
data_source=cfg['data_src'],
physic_unit=cfg['ph_unit'])
def hex_to_rgb(value):
value = value.lstrip('#')
lv = len(value)
return [int(value[i:i + lv // 3], 16) for i in range(0, lv, lv // 3)]
if __name__ == '__main__':
if len(sys.argv) < 2:
print("Usage: " + sys.argv[0] + " MAIA_file ")
sys.exit()
fnmaia = sys.argv[1]
maia_scene = Scene(reader='maia', filenames=[fnmaia])
print(maia_scene.available_dataset_ids())
maia_scene.load(["CloudType", "ct", "cma", "cma_conf",
'opaq_cloud', "CloudTopPres",
"CloudTopTemp", "Alt_surface"])
# CloudType is a bit field containing the actual "ct" with values
# from 0 to 20 which can be interpreted according to the cpool colormap
# "ct" can be display in black and white:
maia_scene.show("ct")
# but it is better to palettize the image:
# step 1: creation of the palette
mycolors = []
for i in range(21):
mycolors.append(hex_to_rgb(cpool[i]))
arr = np.array(mycolors)
np.save("/tmp/binary_maia_ct_colormap.npy", arr)
import os
from satpy import Scene
from datetime import datetime
from satpy.utils import debug_on
import pyninjotiff
from glob import glob
from pyresample.utils import load_area
debug_on()
chn = "airmass"
ninjoRegion = load_area("areas.def", "nrEURO3km")
filenames = glob("data/*__")
global_scene = Scene(reader="hrit_msg", filenames=filenames)
global_scene.load([chn])
local_scene = global_scene.resample(ninjoRegion)
local_scene.save_dataset(chn, filename="airmass.tif", writer='ninjotiff',
sat_id=6300014,
chan_id=6500015,
data_cat='GPRN',
data_source='EUMCAST',
nbits=8)
