def test_sunangles(self):
"""Test the sun-angle calculations."""
lat, lon = 58.6167, 16.1833 # Norrkoping
time_slot = datetime(2011, 9, 23, 12, 0)
sun_theta = astr.sun_zenith_angle(time_slot, lon, lat)
self.assertAlmostEqual(sun_theta, 60.371433482557833, places=8)
sun_theta = astr.sun_zenith_angle(time_slot, 0., 0.)
self.assertAlmostEqual(sun_theta, 1.8751916863323426, places=8)
def test_sunangles(self):
"""Test the sun-angle calculations."""
lat, lon = 58.6167, 16.1833 # Norrkoping
time_slot = datetime(2011, 9, 23, 12, 0)
sun_theta = astr.sun_zenith_angle(time_slot, lon, lat)
self.assertAlmostEqual(sun_theta, 60.371433482557833, places=8)
sun_theta = astr.sun_zenith_angle(time_slot, 0., 0.)
self.assertAlmostEqual(sun_theta, 1.8751916863323426, places=8)
def _get_sunlight_coverage(area_def, start_time, overpass=None):
"""Get the sunlight coverage of *area_def* at *start_time*."""
adp = AreaDefBoundary(area_def, frequency=100).contour_poly
poly = get_twilight_poly(start_time)
if overpass is not None:
ovp = overpass.boundary.contour_poly
cut_area_poly = adp.intersection(ovp)
else:
cut_area_poly = adp
if cut_area_poly is None:
if not adp._is_inside(ovp):
return 0.0
else:
# Should already have been taken care of in pyresample.spherical.intersection
cut_area_poly = adp
daylight = cut_area_poly.intersection(poly)
if daylight is None:
if sun_zenith_angle(start_time, *area_def.get_lonlat(0, 0)) < 90:
return 1.0
else:
return 0.0
else:
daylight_area = daylight.area()
total_area = adp.area()
return daylight_area / total_area
def _get_sunlight_coverage(area_def,
start_time,
sza_threshold=90,
overpass=None):
"""Get the sunlight coverage of *area_def* at *start_time* as a value between 0 and 1."""
adp = polygon_for_area(area_def)
poly = get_twilight_poly(start_time)
if overpass is not None:
ovp = overpass.boundary.contour_poly
cut_area_poly = adp.intersection(ovp)
else:
cut_area_poly = adp
if cut_area_poly is None:
if not adp._is_inside(ovp):
return 0.0
else:
# Should already have been taken care of in pyresample.spherical.intersection
cut_area_poly = adp
daylight = cut_area_poly.intersection(poly)
if daylight is None:
if sun_zenith_angle(start_time, *area_def.get_lonlat(
0, 0)) < sza_threshold:
return 1.0
else:
return 0.0
else:
daylight_area = daylight.area()
total_area = adp.area()
return daylight_area / total_area
def _get_sunlight_coverage(area_def, start_time, overpass=None):
"""Get the sunlight coverage of *area_def* at *start_time* as a value between 0 and 1."""
if area_def.proj_dict.get('proj') == 'geos':
adp = Boundary(*get_geostationary_bounding_box(
area_def, nb_points=100)).contour_poly
else:
adp = AreaDefBoundary(area_def, frequency=100).contour_poly
poly = get_twilight_poly(start_time)
if overpass is not None:
ovp = overpass.boundary.contour_poly
cut_area_poly = adp.intersection(ovp)
else:
cut_area_poly = adp
if cut_area_poly is None:
if not adp._is_inside(ovp):
return 0.0
else:
# Should already have been taken care of in pyresample.spherical.intersection
cut_area_poly = adp
daylight = cut_area_poly.intersection(poly)
if daylight is None:
if sun_zenith_angle(start_time, *area_def.get_lonlat(0, 0)) < 90:
return 1.0
else:
return 0.0
else:
daylight_area = daylight.area()
total_area = cut_area_poly.area()
return daylight_area / total_area
def reflectance_correction(rad_arr, lons, lats):
'''Make satpy "reflectances" consistent with usual definition
Reflectance is normally defined as outgoing radiation/incoming radiation.
In satpy, the denominator is set to a fixed value - the incoming radiation
for a solar zenith angle of 0 and earth-sun distance of one. This function
corrects this using the actual earth-sun distance and the solar zenith
angle appropriate for the observation time. See discussion at
https://github.com/pytroll/satpy/issues/536 for further details.
Args:
rad_arr (Xarray): Xarray for one of the two visible channels (0.6, 0.8) from satpy scene
lons (ndarray, shape(nlat,nlon)): Array of longitude values
lats (ndarray, shape(nlat,nlon)): Array of longitude values
Returns:
rad_arr (Xarray): Input array, with reflectance corrected to account for
solar zenith angle and earth-sun distance.
'''
from astropy.units import AU # Requires virtual environment
from sunpy.sun import sunearth_distance # Requires virtual environment
from pyorbital.astronomy import sun_zenith_angle
nx = rad_arr.sizes['x']
ny = rad_arr.sizes['y']
mu0 = np.ma.zeros((ny,nx))
dist_sun_earth = np.ma.zeros(ny)
Tacq = rad_arr.attrs["end_time"] - rad_arr.attrs["start_time"]
for j in range(ny) :
tacq = rad_arr.attrs["start_time"] + datetime.timedelta( seconds=(j/float(ny))*Tacq.seconds )
mu0[j,:] = np.ma.masked_outside( np.pi*sun_zenith_angle( tacq, lons[j,:], lats[j,:])/180.,0.035, 1, copy=False) # in degrees
dist_sun_earth[j] = float(sunearth_distance(tacq) / AU)
rad_arr.values *= ((dist_sun_earth[:,None]**2) / np.cos(mu0)) # sun earth distance in AU.
return rad_arr
def get_angles(self):
self.get_times()
tle1, tle2 = self.get_tle_lines()
orb = Orbital(self.spacecrafts_orbital[self.spacecraft_id],
line1=tle1,
line2=tle2)
sat_azi, sat_elev = orb.get_observer_look(self.times[:, np.newaxis],
self.lons, self.lats, 0)
sat_zenith = 90 - sat_elev
sun_zenith = astronomy.sun_zenith_angle(self.times[:, np.newaxis],
self.lons, self.lats)
alt, sun_azi = astronomy.get_alt_az(self.times[:, np.newaxis],
self.lons, self.lats)
del alt
sun_azi = np.rad2deg(sun_azi)
sun_azi = np.where(sun_azi < 0, sun_azi + 180, sun_azi - 180)
rel_azi = abs(sat_azi - sun_azi)
rel_azi = np.where(rel_azi > 180.0, 360.0 - rel_azi, rel_azi)
return sat_azi, sat_zenith, sun_azi, sun_zenith, rel_azi
def sza(self):
#self.check_channels("VIS006", "HRV", "IR_108")
import numpy as np
# calculate longitude/latitude and solar zenith angle
from pyorbital.astronomy import sun_zenith_angle
lonlats = self.area.get_lonlats()
#lonlats = self["IR_108"].area.get_lonlats()
sza = sun_zenith_angle(self.time_slot, lonlats[0], lonlats[1])
### stupid numbers for outside disk!!!
#import numpy.ma as ma
#sza=ma.masked_equal(sza, 8.24905797976)
#sza=ma.masked_equal(sza, 8.25871138053)
img = GeoImage(sza,
self.area,
self.time_slot,
fill_value=(0, 0, 0),
mode="L")
from trollimage.colormap import rainbow
cm = deepcopy(rainbow)
cm.set_range(0, 90)
img.colorize(cm)
return img
def __call__(self, projectables, optional_datasets=None, **info):
"""Get the corrected reflectance when removing Rayleigh scattering. Uses
pyspectral.
"""
from pyspectral.rayleigh import Rayleigh
(vis, red) = projectables
if vis.shape != red.shape:
raise IncompatibleAreas
try:
(sata, satz, suna, sunz) = optional_datasets
except ValueError:
from pyorbital.astronomy import get_alt_az, sun_zenith_angle
from pyorbital.orbital import get_observer_look
lons, lats = vis.info['area'].get_lonlats()
sunalt, suna = get_alt_az(vis.info['start_time'], lons, lats)
suna = np.rad2deg(suna)
sunz = sun_zenith_angle(vis.info['start_time'], lons, lats)
sata, satel = get_observer_look(vis.info['satellite_longitude'],
vis.info['satellite_latitude'],
vis.info['satellite_altitude'],
vis.info['start_time'], lons, lats,
0)
satz = 90 - satel
del satel
LOG.info('Removing Rayleigh scattering and aerosol absorption')
# First make sure the two azimuth angles are in the range 0-360:
sata = np.mod(sata, 360.)
suna = np.mod(suna, 360.)
ssadiff = np.abs(suna - sata)
ssadiff = np.where(ssadiff > 180, 360 - ssadiff, ssadiff)
del sata, suna
atmosphere = self.info.get('atmosphere', 'us-standard')
aerosol_type = self.info.get('aerosol_type', 'marine_clean_aerosol')
corrector = Rayleigh(vis.info['platform_name'],
vis.info['sensor'],
atmosphere=atmosphere,
aerosol_type=aerosol_type)
try:
refl_cor_band = corrector.get_reflectance(sunz, satz, ssadiff,
vis.id.name, red)
except KeyError:
LOG.warning(
"Could not get the reflectance correction using band name: %s",
vis.id.name)
LOG.warning(
"Will try use the wavelength, however, this may be ambiguous!")
refl_cor_band = corrector.get_reflectance(sunz, satz, ssadiff,
vis.id.wavelength[1],
red)
proj = Dataset(vis - refl_cor_band, copy=False, **vis.info)
self.apply_modifier_info(vis, proj)
return proj
def bad_sunzen_range(area, product_config, area_id, prod, time_slot):
"""Check if Sun zenith angle is valid at the configured location."""
product_conf = product_config["product_list"][area_id]["products"][prod]
if ("sunzen_night_minimum" not in product_conf
and "sunzen_day_maximum" not in product_conf):
return False
if astronomy is None:
LOGGER.warning("Pyorbital not installed, unable to calculate "
"Sun zenith angles!")
return False
if area.lons is None:
area.lons, area.lats = area.get_lonlats()
lon, lat = None, None
try:
lon = product_conf["sunzen_lon"]
lat = product_conf["sunzen_lat"]
except KeyError:
pass
if lon is None or lat is None:
try:
x_idx = product_conf["sunzen_x_idx"]
y_idx = product_conf["sunzen_y_idx"]
lon = area.lons[x_idx]
lat = area.lats[y_idx]
except KeyError:
pass
if lon is None or lat is None:
LOGGER.info("Using area center for Sun zenith angle calculation")
y_idx = int(area.y_size / 2)
x_idx = int(area.x_size / 2)
lon, lat = area.get_lonlat(y_idx, x_idx)
sunzen = astronomy.sun_zenith_angle(time_slot, lon, lat)
LOGGER.debug("Sun zenith angle is %.2f degrees", sunzen)
try:
limit = product_conf["sunzen_night_minimum"]
if sunzen < limit:
return True
else:
return False
except KeyError:
pass
try:
limit = product_conf["sunzen_day_maximum"]
if sunzen > limit:
return True
else:
return False
except KeyError:
pass
def parse_jaxa_hotspot_txt(self, file_path):
"""
Parse the JAXA Himawari-8/9 AHI hotspot text file and insert attrs into the Pandas DataFrame
Args:
file_path (str): File path to the JAXA Himawari-8/9 hotspot .csv
Returns:
hs_df (obj): DataFrame obj that contains attr of Himawari-8/9 hospot
"""
hs_ahi_df = pd.DataFrame()
cols_to_use_list = [0, 2, *(i for i in range(7, 24))]
date_col_list = [0]
names_list = ['date', 'satellite', 'lon', 'lat', 'viewzenang', 'viewazang', 'pixwid', 'pixlen', 't07',
't14', 't07_t14', 'meant07', 'meant14', 'meandt', 'sdt07', 'sdt14', 'sddt',
'ref3', 'ref4']
for file in glob.glob(file_path):
try:
log.debug(f'Reading {file}')
temp_hs_ahi_df = pd.read_csv(file, sep=",", skiprows=[0], \
header=None, usecols=cols_to_use_list, \
names=names_list, \
parse_dates=date_col_list)
temp_hs_ahi_df['satellite'] = 'Himawari-8/9'
except Exception as e:
log.error(f'Error reading {file}')
log.error(e)
continue
if len(temp_hs_ahi_df) > 0:
try:
temp_hs_ahi_df['solarazang'] = temp_hs_ahi_df.apply( \
lambda x: np.degrees(astronomy.get_alt_az(x['date'], x['lon'], x['lat'])[1]), axis=1)
temp_hs_ahi_df['solarzenang'] = temp_hs_ahi_df.apply( \
lambda x: astronomy.sun_zenith_angle(x['date'], x['lon'], x['lat']), axis=1)
temp_hs_ahi_df['relazang'] = temp_hs_ahi_df['solarazang'] - temp_hs_ahi_df['viewazang']
temp_hs_ahi_df['sunglint_angle'] = temp_hs_ahi_df.apply(compute_sun_glint_angle, axis=1)
except Exception as e:
temp_hs_ahi_df['sunglint_angle'] = np.nan
try:
temp_hs_ahi_df.loc[(temp_hs_ahi_df['date'].dt.hour >= 0) \
& (temp_hs_ahi_df['date'].dt.hour <= 11), 'daynight'] = 'day'
temp_hs_ahi_df.loc[temp_hs_ahi_df['date'].dt.hour > 11, \
'daynight'] = 'night'
temp_hs_ahi_df['date'] = temp_hs_ahi_df['date'].dt.strftime( \
"%d/%m/%Y %H:%M:%S")
except Exception as e:
date_from_file = datetime.strptime(filename[4:17], "%Y%m%d_%H%M")
temp_hs_ahi_df['date'] = date_from_file.strftime("%d/%m/%Y %H:%M:%S")
hs_ahi_df = pd.concat([hs_ahi_df, temp_hs_ahi_df])
if len(hs_ahi_df) > 0:
hs_ahi_df = hs_ahi_df.reset_index(drop=True)
self.hs_df = pd.concat([hs_ahi_df, self.hs_df])
def get_sza_mask(self, sza_max=80):
# calculate longitude/latitude and solar zenith angle
from pyorbital.astronomy import sun_zenith_angle
lonlats = self.area.get_lonlats()
sza = sun_zenith_angle(self.time_slot, lonlats[0], lonlats[1])
mask = np.array(sza < sza_max)
return mask
def add_sunzenith(self):
"""Derive the sun zenith angles and add to data series"""
from pyorbital import astronomy
sunz = []
for idx in self.data.index:
sunz.append(astronomy.sun_zenith_angle(self.data.loc[idx, 'date'],
self.data.loc[idx, 'lon'],
self.data.loc[idx, 'lat']))
self.data['sunz'] = pd.Series(sunz, index=self.data.index)
def add_sunzenith(self):
"""Derive the sun zenith angles and add to data series"""
from pyorbital import astronomy
sunz = []
for idx in self.data.index:
sunz.append(
astronomy.sun_zenith_angle(self.data.loc[idx, 'date'],
self.data.loc[idx, 'lon'],
self.data.loc[idx, 'lat']))
self.data['sunz'] = pd.Series(sunz, index=self.data.index)
def _get_day_percentage(self, refl_swath):
swath_name = refl_swath["swath_definition"]["swath_name"]
if swath_name not in self._day_percentage:
from pyorbital import astronomy
lons = refl_swath["swath_definition"].get_longitude_array()
lats = refl_swath["swath_definition"].get_latitude_array()
invalid_mask = refl_swath.get_data_mask()
sza_data = astronomy.sun_zenith_angle(refl_swath["begin_time"], lons, lats)
valid_day_mask = (sza_data < self.sza_threshold) & ~invalid_mask
fraction_day = numpy.count_nonzero(valid_day_mask) / (float(sza_data.size) - numpy.count_nonzero(invalid_mask))
self._day_percentage[swath_name] = fraction_day * 100.0
else:
LOG.debug("Using cached day percentage")
return self._day_percentage[swath_name]
def get_angles(self, vis):
from pyorbital.astronomy import get_alt_az, sun_zenith_angle
from pyorbital.orbital import get_observer_look
lons, lats = vis.attrs['area'].get_lonlats_dask(chunks=vis.data.chunks)
suna = get_alt_az(vis.attrs['start_time'], lons, lats)[1]
suna = xu.rad2deg(suna)
sunz = sun_zenith_angle(vis.attrs['start_time'], lons, lats)
sata, satel = get_observer_look(vis.attrs['satellite_longitude'],
vis.attrs['satellite_latitude'],
vis.attrs['satellite_altitude'],
vis.attrs['start_time'], lons, lats, 0)
satz = 90 - satel
return sata, satz, suna, sunz
def _get_day_percentage(self, refl_swath):
swath_name = refl_swath["swath_definition"]["swath_name"]
if swath_name not in self._day_percentage:
from pyorbital import astronomy
lons = refl_swath["swath_definition"].get_longitude_array()
lats = refl_swath["swath_definition"].get_latitude_array()
invalid_mask = refl_swath.get_data_mask()
sza_data = astronomy.sun_zenith_angle(refl_swath["begin_time"], lons, lats)
valid_day_mask = (sza_data < self.sza_threshold) & ~invalid_mask
fraction_day = numpy.count_nonzero(valid_day_mask) / (float(sza_data.size) - numpy.count_nonzero(invalid_mask))
self._day_percentage[swath_name] = fraction_day * 100.0
else:
LOG.debug("Using cached day percentage")
return self._day_percentage[swath_name]
def __call__(self, projectables, optional_datasets=None, **info):
"""Get the corrected reflectance when removing Rayleigh scattering. Uses
pyspectral.
"""
from pyspectral.rayleigh import Rayleigh
(vis, blue) = projectables
if vis.shape != blue.shape:
raise IncompatibleAreas
try:
(sata, satz, suna, sunz) = optional_datasets
except ValueError:
from pyorbital.astronomy import get_alt_az, sun_zenith_angle
from pyorbital.orbital import get_observer_look
sunalt, suna = get_alt_az(
vis.info['start_time'], *vis.info['area'].get_lonlats())
suna = np.rad2deg(suna)
sunz = sun_zenith_angle(
vis.info['start_time'], *vis.info['area'].get_lonlats())
lons, lats = vis.info['area'].get_lonlats()
sata, satel = get_observer_look(vis.info['satellite_longitude'],
vis.info['satellite_latitude'],
vis.info['satellite_altitude'],
vis.info['start_time'],
lons, lats, 0)
satz = 90 - satel
del satel
LOG.info('Removing Rayleigh scattering and aerosol absorption')
ssadiff = np.abs(suna - sata)
ssadiff = np.where(ssadiff > 180, 360 - ssadiff, ssadiff)
del sata, suna
atmosphere = self.info.get('atmosphere', 'us-standard')
aerosol_type = self.info.get('aerosol_type', 'marine_clean_aerosol')
corrector = Rayleigh(vis.info['platform_name'], vis.info['sensor'],
atmosphere=atmosphere,
aerosol_type=aerosol_type)
refl_cor_band = corrector.get_reflectance(
sunz, satz, ssadiff, vis.id.wavelength[1], blue)
proj = Dataset(vis - refl_cor_band,
copy=False,
**vis.info)
self.apply_modifier_info(vis, proj)
return proj
def get_angles(self):
"""Get azimuth and zenith angles.
Azimuth angle definition is the same as in pyorbital, but with
different units (degrees not radians for sun azimuth angles)
and different ranges.
Returns:
sat_azi: satellite azimuth angle
degree clockwise from north in range ]-180, 180],
sat_zentih: satellite zenith angles in degrees in range [0,90],
sun_azi: sun azimuth angle
degree clockwise from north in range ]-180, 180],
sun_zentih: sun zenith angles in degrees in range [0,90],
rel_azi: absolute azimuth angle difference in degrees between sun
and sensor in range [0, 180]
"""
self.get_times()
self.get_lonlat()
tle1, tle2 = self.get_tle_lines()
times = self.times
orb = Orbital(self.spacecrafts_orbital[self.spacecraft_id],
line1=tle1,
line2=tle2)
sat_azi, sat_elev = orb.get_observer_look(times[:, np.newaxis],
self.lons, self.lats, 0)
sat_zenith = 90 - sat_elev
sun_zenith = astronomy.sun_zenith_angle(times[:, np.newaxis],
self.lons, self.lats)
alt, sun_azi = astronomy.get_alt_az(times[:, np.newaxis], self.lons,
self.lats)
del alt
sun_azi = np.rad2deg(sun_azi)
rel_azi = get_absolute_azimuth_angle_diff(sun_azi, sat_azi)
# Scale angles range to half open interval ]-180, 180]
sat_azi = centered_modulus(sat_azi, 360.0)
sun_azi = centered_modulus(sun_azi, 360.0)
# Mask corrupt scanlines
for arr in (sat_azi, sat_zenith, sun_azi, sun_zenith, rel_azi):
arr[self.mask] = np.nan
return sat_azi, sat_zenith, sun_azi, sun_zenith, rel_azi
def sza_check(job):
"""Remove products which are not valid for the current Sun zenith angle."""
scn_mda = _get_scene_metadata(job)
scn_mda.update(job['input_mda'])
start_time = scn_mda['start_time']
product_list = job['product_list']
areas = list(product_list['product_list']['areas'].keys())
for area in areas:
products = list(
product_list['product_list']['areas'][area]['products'].keys())
for product in products:
prod_path = "/product_list/areas/%s/products/%s" % (area, product)
lon = get_config_value(product_list, prod_path, "sunzen_check_lon")
lat = get_config_value(product_list, prod_path, "sunzen_check_lat")
if lon is None or lat is None:
LOG.debug(
"No 'sunzen_check_lon' or 'sunzen_check_lat' configured, "
"can\'t check Sun elevation for %s / %s", area, product)
continue
sunzen = sun_zenith_angle(start_time, lon, lat)
LOG.debug("Sun zenith angle is %.2f degrees", sunzen)
# Check nighttime limit
limit = get_config_value(product_list, prod_path,
"sunzen_minimum_angle")
if limit is not None:
if sunzen < limit:
LOG.info(
"Sun zenith angle to small for nighttime "
"product '%s', product removed.", product)
dpath.util.delete(product_list, prod_path)
continue
# Check daytime limit
limit = get_config_value(product_list, prod_path,
"sunzen_maximum_angle")
if limit is not None:
if sunzen > limit:
LOG.info(
"Sun zenith angle too large for daytime "
"product '%s', product removed.", product)
dpath.util.delete(product_list, prod_path)
continue
if len(product_list['product_list']['areas'][area]['products']) == 0:
LOG.info("Removing empty area: %s", area)
dpath.util.delete(product_list, '/product_list/areas/%s' % area)
def sunz_filter(pdf, sunz_range):
"""Filter the data according to the sun zenith angle"""
if not 'sunz' in pdf.keys():
from pyorbital import astronomy
idx_selected = []
for idx in pdf.index:
sunz = astronomy.sun_zenith_angle(pdf.loc[idx, 'date'],
pdf.loc[idx, 'lon'],
pdf.loc[idx, 'lat'])
if sunz > sunz_range[0] and sunz < sunz_range[1]:
idx_selected.append(idx)
return pdf.loc[idx_selected, :]
else:
pdf = pdf[pdf['sunz'] > sunz_range[0]]
pdf = pdf[pdf['sunz'] < sunz_range[01]]
return pdf
def bad_sunzen_range_satpy(product_config, group, composite, start_time):
"""Check if Sun zenith angle is valid at the configured location.
SatPy version.
TODO: refactor with bad_sunzen_range()
"""
# FIXME: check all areas within the group
area_id = product_config['groups'][group][0]
product_conf = \
product_config["product_list"][area_id]["products"][composite]
if ("sunzen_night_minimum" not in product_conf
and "sunzen_day_maximum" not in product_conf):
return False
if astronomy is None:
LOGGER.warning("Pyorbital not installed, unable to calculate "
"Sun zenith angles!")
return False
if "sunzen_lon" not in product_conf and "sunzen_lat" not in product_conf:
LOGGER.warning("No 'sunzen_lon' or 'sunzen_lat' configured, "
"can\'t check Sun elevation.")
return False
lon = product_conf["sunzen_lon"]
lat = product_conf["sunzen_lat"]
sunzen = astronomy.sun_zenith_angle(start_time, lon, lat)
LOGGER.debug("Sun zenith angle is %.2f degrees", sunzen)
try:
limit = product_conf["sunzen_night_minimum"]
if sunzen < limit:
return True
else:
return False
except KeyError:
pass
try:
limit = product_conf["sunzen_day_maximum"]
if sunzen > limit:
return True
else:
return False
except KeyError:
pass
def sunz_filter(pdf, sunz_range):
"""Filter the data according to the sun zenith angle"""
if not 'sunz' in pdf.keys():
from pyorbital import astronomy
idx_selected = []
for idx in pdf.index:
sunz = astronomy.sun_zenith_angle(pdf.loc[idx, 'date'],
pdf.loc[idx, 'lon'],
pdf.loc[idx, 'lat'])
if sunz > sunz_range[0] and sunz < sunz_range[1]:
idx_selected.append(idx)
return pdf.loc[idx_selected, :]
else:
pdf = pdf[pdf['sunz'] > sunz_range[0]]
pdf = pdf[pdf['sunz'] < sunz_range[01]]
return pdf
def getSentinel2Geometry(startDateUTC, lengthDays, lat, lon, alt=0.0, mission="Sentinel-2a",
tleFile="../TLE/norad_resource_tle.txt"):
"""Calculate approximate geometry for Sentinel overpasses.
Approximate because it assumes maximum satellite elevation
is the time at which target is imaged.
:param startDateUTC: a datetime object specifying when to start prediction.
:type startDateUTC: object
:param lengthDays: number of days over which to perform calculations.
:type lengthDays: int
:param lat: latitude of target.
:type lat: float
:param lon: longitude of target.
:type lon: float
:param alt: altitude of target (in km).
:type alt: float
:param mission: mission name as in TLE file.
:type mission: str
:param tleFile: TLE file.
:type tleFile: str
:return: a python list containing instances of the sensorGeometry class arranged in date order.
:rtype: list
"""
orb = Orbital(mission, tleFile)
passes = orb.get_next_passes(startDateUTC, 24 * lengthDays, lon, lat, alt)
geomList = []
for p in passes:
look = orb.get_observer_look(p[2], lon, lat, alt)
vza = 90 - look[1]
vaa = look[0]
sza = ast.sun_zenith_angle(p[2], lon, lat)
saa = np.rad2deg(ast.get_alt_az(p[2], lon, lat)[1])
if sza < 90 and vza < 10.3:
thisGeom = sensorGeometry()
thisGeom.date_utc = p[2]
thisGeom.vza = vza
thisGeom.vaa = vaa
thisGeom.sza = sza
thisGeom.saa = saa
geomList.append(thisGeom)
return geomList
def _get_sun_zenith_from_provided_data(projectables, optional_datasets):
"""Get the sunz from available data or compute it if unavailable."""
sun_zenith = None
for dataset in optional_datasets:
if dataset.attrs.get("standard_name") == "solar_zenith_angle":
sun_zenith = dataset.data
if sun_zenith is None:
if sun_zenith_angle is None:
raise ImportError(
"Module pyorbital.astronomy needed to compute sun zenith angles."
)
_nir = projectables[0]
lons, lats = _nir.attrs["area"].get_lonlats(
chunks=_nir.data.chunks)
sun_zenith = sun_zenith_angle(_nir.attrs['start_time'], lons, lats)
return sun_zenith
def get_angles(self, vis):
from pyorbital.astronomy import get_alt_az, sun_zenith_angle
from pyorbital.orbital import get_observer_look
lons, lats = vis.attrs['area'].get_lonlats_dask(
chunks=vis.data.chunks)
sunalt, suna = get_alt_az(vis.attrs['start_time'], lons, lats)
suna = xu.rad2deg(suna)
sunz = sun_zenith_angle(vis.attrs['start_time'], lons, lats)
sata, satel = get_observer_look(
vis.attrs['satellite_longitude'],
vis.attrs['satellite_latitude'],
vis.attrs['satellite_altitude'],
vis.attrs['start_time'],
lons, lats, 0)
satz = 90 - satel
return sata, satz, suna, sunz
def get_angles(self, vis):
"""Get sun and satellite angles to use in crefl calculations."""
from pyorbital.astronomy import get_alt_az, sun_zenith_angle
from pyorbital.orbital import get_observer_look
lons, lats = vis.attrs['area'].get_lonlats(chunks=vis.data.chunks)
lons = da.where(lons >= 1e30, np.nan, lons)
lats = da.where(lats >= 1e30, np.nan, lats)
suna = get_alt_az(vis.attrs['start_time'], lons, lats)[1]
suna = np.rad2deg(suna)
sunz = sun_zenith_angle(vis.attrs['start_time'], lons, lats)
sat_lon, sat_lat, sat_alt = get_satpos(vis)
sata, satel = get_observer_look(
sat_lon,
sat_lat,
sat_alt / 1000.0, # km
vis.attrs['start_time'],
lons, lats, 0)
satz = 90 - satel
return sata, satz, suna, sunz
def get_angles(self, vis):
from pyorbital.astronomy import get_alt_az, sun_zenith_angle
from pyorbital.orbital import get_observer_look
lons, lats = vis.attrs['area'].get_lonlats_dask(chunks=vis.data.chunks)
suna = get_alt_az(vis.attrs['start_time'], lons, lats)[1]
suna = np.rad2deg(suna)
sunz = sun_zenith_angle(vis.attrs['start_time'], lons, lats)
sat_lon, sat_lat, sat_alt = get_satpos(vis)
sata, satel = get_observer_look(
sat_lon,
sat_lat,
sat_alt / 1000.0, # km
vis.attrs['start_time'],
lons,
lats,
0)
satz = 90 - satel
return sata, satz, suna, sunz
def __call__(self, projectables, optional_datasets=None, **info):
"""Get the corrected reflectance when removing Rayleigh scattering. Uses
pyspectral.
"""
from pyspectral.rayleigh import Rayleigh
(vis, ) = projectables
try:
(sata, satz, suna, sunz) = optional_datasets
except ValueError:
from pyorbital.astronomy import get_alt_az, sun_zenith_angle
from pyorbital.orbital import get_observer_look
sunalt, suna = get_alt_az(vis.info['start_time'],
*vis.info['area'].get_lonlats())
sunz = sun_zenith_angle(vis.info['start_time'],
*vis.info['area'].get_lonlats())
lons, lats = vis.info['area'].get_lonlats()
sata, satel = get_observer_look(vis.info['satellite_longitude'],
vis.info['satellite_latitude'],
vis.info['satellite_altitude'],
vis.info['start_time'], lons, lats,
0)
satz = 90 - satel
LOG.info('Removing Rayleigh scattering')
ssadiff = np.abs(suna - sata)
ssadiff = np.where(np.greater(ssadiff, 180), 360 - ssadiff, ssadiff)
corrector = Rayleigh(vis.info['platform_name'],
vis.info['sensor'],
atmosphere='us-standard',
rural_aerosol=False)
refl_cor_band = corrector.get_reflectance(sunz, satz, ssadiff,
vis.info['id'].wavelength[1],
vis)
proj = Projectable(vis - refl_cor_band, copy=False, **vis.info)
self.apply_modifier_info(vis, proj)
return proj
def __call__(self, projectables, optional_datasets=None, **info):
"""Get the corrected reflectance when removing Rayleigh scattering. Uses
pyspectral.
"""
from pyspectral.rayleigh import Rayleigh
(vis,) = projectables
try:
(sata, satz, suna, sunz) = optional_datasets
except ValueError:
from pyorbital.astronomy import get_alt_az, sun_zenith_angle
from pyorbital.orbital import get_observer_look
sunalt, suna = get_alt_az(
vis.info['start_time'], *vis.info['area'].get_lonlats())
sunz = sun_zenith_angle(
vis.info['start_time'], *vis.info['area'].get_lonlats())
lons, lats = vis.info['area'].get_lonlats()
sata, satel = get_observer_look(vis.info['satellite_longitude'],
vis.info['satellite_latitude'],
vis.info['satellite_altitude'],
vis.info['start_time'],
lons, lats, 0)
satz = 90 - satel
LOG.info('Removing Rayleigh scattering')
ssadiff = np.abs(suna - sata)
ssadiff = np.where(np.greater(ssadiff, 180), 360 - ssadiff, ssadiff)
corrector = Rayleigh(
vis.info['platform_name'], vis.info['sensor'], atmosphere='us-standard', rural_aerosol=False)
refl_cor_band = corrector.get_reflectance(
sunz, satz, ssadiff, vis.info['id'].wavelength[1], vis)
proj = Projectable(vis - refl_cor_band,
copy=False,
**vis.info)
self.apply_modifier_info(vis, proj)
return proj
def HRVFog(self, downscale=False, return_data=False):
"""Make an HRV RGB image composite.
+--------------------+--------------------+--------------------+
| Channels | Temp | Gamma |
+====================+====================+====================+
| NIR1.6 | 0 - 70 | gamma 1 |
+--------------------+--------------------+--------------------+
| HRV | 0 - 100 | gamma 1 |
+--------------------+--------------------+--------------------+
| HRV | 0 - 100 | gamma 1 |
+--------------------+--------------------+--------------------+
"""
self.check_channels(1.6, "HRV")
from pyorbital.astronomy import sun_zenith_angle
lonlats = self["HRV"].area.get_lonlats()
sza = sun_zenith_angle(self.time_slot, lonlats[0], lonlats[1])
cos_sza = np.cos(np.radians(sza)) + 0.05
ch1 = self[1.6].data / cos_sza
ch2 = self["HRV"].data / cos_sza
ch3 = self["HRV"].data / cos_sza
# this area exception is not nice!
if downscale or (self["HRV"].area.name == 'ccs4' or self["HRV"].area.name
== 'Switzerland_stereographic_500m'):
print('... downscale NIR1.6')
from plot_coalition2 import downscale_array
ch1 = downscale_array(ch1)
if return_data:
return ch1, ch2, ch3
img = GeoImage((ch1, ch2, ch3),
self.area,
self.time_slot,
fill_value=(0, 0, 0),
mode="RGB",
crange=((0, 70), (0, 100), (0, 100)))
return img
def _get_sunlight_coverage(area_def,
start_time,
sza_threshold=90,
overpass=None):
"""Get the sunlight coverage of *area_def* at *start_time* as a value between 0 and 1."""
if isinstance(area_def, SwathDefinition):
lons, lats = area_def.get_lonlats()
freq = int(lons.shape[-1] * 0.10)
lons, lats = da.compute(lons[::freq, ::freq], lats[::freq, ::freq])
adp = Boundary(lons.ravel(), lats.ravel()).contour_poly
elif area_def.is_geostationary:
adp = Boundary(*get_geostationary_bounding_box(
area_def, nb_points=100)).contour_poly
else:
adp = AreaDefBoundary(area_def, frequency=100).contour_poly
poly = get_twilight_poly(start_time)
if overpass is not None:
ovp = overpass.boundary.contour_poly
cut_area_poly = adp.intersection(ovp)
else:
cut_area_poly = adp
if cut_area_poly is None:
if not adp._is_inside(ovp):
return 0.0
else:
# Should already have been taken care of in pyresample.spherical.intersection
cut_area_poly = adp
daylight = cut_area_poly.intersection(poly)
if daylight is None:
if sun_zenith_angle(start_time, *area_def.get_lonlat(
0, 0)) < sza_threshold:
return 1.0
else:
return 0.0
else:
daylight_area = daylight.area()
total_area = adp.area()
return daylight_area / total_area
def get_solar_angles(scene, lons, lats):
"""Compute solar angles.
Compute angles for each scanline using their acquisition time to account for
the earth's rotation over the course of one scan.
Returns:
Solar azimuth angle, Solar zenith angle in degrees
"""
suna = np.full(lons.shape, np.nan)
sunz = np.full(lons.shape, np.nan)
mean_acq_time = get_mean_acq_time(scene)
for line, acq_time in enumerate(mean_acq_time.values):
if np.isnat(acq_time):
continue
_, suna_line = get_alt_az(acq_time, lons[line, :], lats[line, :])
suna_line = np.rad2deg(suna_line)
suna[line, :] = suna_line
sunz[line, :] = sun_zenith_angle(acq_time, lons[line, :],
lats[line, :])
return suna, sunz
def _get_measurements_from_hrit(hrit_files, year, month, day, slot):
""" Read SEVIRI HRIT slot and calculate solar zenith angle for RGB. """
scene = satpy.Scene(reader="seviri_l1b_hrit", filenames=hrit_files)
scene.load(['VIS006', 'VIS008', 'IR_016', 'IR_108'])
utc_time = datetime(int(year), int(month), int(day), int(slot[:2]),
int(slot[2:]))
lon, lat = scene['VIS006'].attrs['area'].get_lonlats()
sunzen = astronomy.sun_zenith_angle(utc_time, lon, lat)
data = {
'VIS006': scene['VIS006'].values,
'VIS008': scene['VIS008'].values,
'IR_016': scene['IR_016'].values,
'IR_108': scene['IR_108'].values,
'sunzen': sunzen,
'lon': lon,
'lat': lat
}
print(' READ HRIT DATA')
return data, hrit_files
def fogpy(chn108, chn39, chn08, chn16, chn06, chn87, chn120, time, lat, lon,
elevation, cot, reff):
""" The fog and low stratus detection and forecasting algorithms are
utilizing the methods proposed in different innovative studies:
Arguements:
chn108 Array for the 10.8 μm channel
chn39 Array for the 3.9 μm channel
chn08 Array for the 0.8 μm channel
chn16 Array for the 1.6 μm channel
chn06 Array for the 0.6 μm channel
chn87 Array for the 8.7 μm channel
chn120 Array for the 12.0 μm channel
time Datetime object for the satellite scence
lat Array of latitude values
lon Array of longitude values
elevation Array of area elevation
cot Array of cloud optical thickness (depth)
reff Array of cloud particle effective raduis
Returns:
Infrared image with fog mask
- A novel approach to fog/low stratus detection using Meteosat 8 data
J. Cermak & J. Bendix
- Detecting ground fog from space – a microphysics-based approach
J. Cermak & J. Bendix
The algorithm can be applied to satellite zenith angle lower than 70°
and a maximum solar zenith angle of 80°.
The algorithm workflow is a succession of differnt masking approaches
from coarse to finer selection to find fog and low stratus clouds within
provided satellite images.
Input: Calibrated satellite images >-----
|
1. Cloud masking -------------------
|
2. Spatial clustering---------------
|
3. Maximum margin elevation --------
|
4. Surface homogenity check --------
|
5. Microphysics plausibility check -
|
6. Differenciate fog - low status --
|
7. Fog dissipation -----------------
|
8. Nowcasting ----------------------
|
Output: fog and low stratus mask <--
"""
arrsize = chn108.size
# Dictionary of filtered values.
filters = {}
prev = 0
# 1. Cloud masking
logger.info("### Applying fog cloud filters to input array ###")
# Given the combination of a solar and a thermal signal at 3.9 μm,
# the difference in radiances to the 10.8 μm must be larger for a
# cloud-contaminated pixel than for a clear pixel:
cm_diff = chn108 - chn39
# In the histogram of the difference the clear sky peak is identified
# within a certain range. The nearest significant relative minimum in the
# histogram towards more negative values is detected and used as a
# threshold to separate clear from cloudy pixels in the image.
# Create histogram
hist = (np.histogram(cm_diff.compressed(), bins='auto'))
# Find local min and max values
localmin = (np.diff(np.sign(np.diff(hist[0]))) > 0).nonzero()[0] + 1
localmax = (np.diff(np.sign(np.diff(hist[0]))) < 0).nonzero()[0] + 1
# Utilize scipy signal funciton to find peaks
peakind = find_peaks_cwt(hist[0], np.arange(1, len(hist[1]) / 10))
peakrange = hist[1][peakind][(hist[1][peakind] >= -10) &
(hist[1][peakind] < 10)]
minpeak = np.min(peakrange)
maxpeak = np.max(peakrange)
# Determine threshold
logger.debug("Histogram range for cloudy/clear sky pixels: {} - {}"
.format(minpeak, maxpeak))
#plt.bar(hist[1][:-1], hist[0])
#plt.title("Histogram with 'auto' bins")
#plt.show()
thres = np.max(hist[1][localmin[(hist[1][localmin] <= maxpeak) &
(hist[1][localmin] >= minpeak) &
(hist[1][localmin] < 0.5)]])
if thres > 0 or thres < -5:
logger.warning("Cloud maks difference threshold {} outside normal"
" range (from -5 to 0)".format(thres))
else:
logger.debug("Cloud mask difference threshold set to %s" % thres)
# Create cloud mask for image array
cloud_mask = cm_diff > thres
filters['cloud'] = np.nansum(cloud_mask)
prev += filters['cloud']
fog_mask = cloud_mask
# Remove remaining snow pixels
chn108_ma = np.ma.masked_where(cloud_mask, chn108)
chn16_ma = np.ma.masked_where(cloud_mask, chn16)
chn08_ma = np.ma.masked_where(cloud_mask, chn08)
chn06_ma = np.ma.masked_where(cloud_mask, chn06)
# Snow has a certain minimum reflectance (0.11 at 0.8 μm) and snow has a
# certain minimum temperature (256 K)
# Snow displays a lower reflectivity than water clouds at 1.6 μm, combined
# with a slightly higher level of absorption (Wiscombe and Warren, 1980)
# Calculate Normalized Difference Snow Index
ndsi = (chn06 - chn16) / (chn06 + chn16)
# Where the NDSI exceeds a certain threshold (0.4) and the two other
# criteria are met, a pixel is rejected as snow-covered.
snow_mask = (chn08 / 100 >= 0.11) & (chn108 >= 256) & (ndsi >= 0.4)
fog_mask = fog_mask | snow_mask
filters['snow'] = np.nansum(fog_mask) - prev
prev += filters['snow']
# Ice cloud exclusion
# Only warm fog (i.e. clouds in the water phase) are considered.
# No ice fog!!!
# Difference of brightness temperatures in the 12.0 and 8.7 μm channels
# is used as an indicator of cloud phase (Strabala et al., 1994).
# Where it exceeds 2.5 K, a water-cloud-covered pixel is assumed with a
# large degree of certainty.
chn120_ma = np.ma.masked_where(cloud_mask | snow_mask, chn120)
chn108_ma = np.ma.masked_where(cloud_mask | snow_mask, chn108)
chn87_ma = np.ma.masked_where(cloud_mask | snow_mask, chn87)
ic_diff = chn120 - chn87
# Straightforward temperature test, cutting off at very low 10.8 μm
# brightness temperatures (250 K).
# Create ice cloud mask
ice_mask = (ic_diff < 2.5) | (chn108 < 250)
fog_mask = fog_mask | ice_mask
filters['ice'] = np.nansum(fog_mask) - prev
prev += filters['ice']
# Thin cirrus is detected by means of the split-window IR channel
# brightness temperature difference (T10.8 –T12.0 ). This difference is
# compared to a threshold dynamically interpolated from a lookup table
# based on satellite zenith angle and brightness temperature at 10.8 μm
# (Saunders and Kriebel, 1988)
chn120_ma = np.ma.masked_where(cloud_mask | snow_mask | ice_mask, chn120)
chn108_ma = np.ma.masked_where(cloud_mask | snow_mask | ice_mask, chn108)
bt_diff = chn108 - chn120
# Calculate sun zenith angles
sza = astronomy.sun_zenith_angle(time, lon, lat)
minsza = np.min(sza)
maxsza = np.max(sza)
logger.debug("Found solar zenith angles from %s to %s°" % (minsza,
maxsza))
# Calculate secant of sza
# secsza = np.ma.masked_where(cloud_mask | snow_mask | ice_mask,
# (1 / np.cos(np.deg2rad(sza))))
secsza = 1 / np.cos(np.deg2rad(sza))
# Lookup table for BT difference thresholds at certain sec(sun zenith
# angles) and 10.8 μm BT
lut = {260: {1.0: 0.55, 1.25: 0.60, 1.50: 0.65, 1.75: 0.90, 2.0: 1.10},
270: {1.0: 0.58, 1.25: 0.63, 1.50: 0.81, 1.75: 1.03, 2.0: 1.13},
280: {1.0: 1.30, 1.25: 1.61, 1.50: 1.88, 1.75: 2.14, 2.0: 2.30},
290: {1.0: 3.06, 1.25: 3.72, 1.50: 3.95, 1.75: 4.27, 2.0: 4.73},
300: {1.0: 5.77, 1.25: 6.92, 1.50: 7.00, 1.75: 7.42, 2.0: 8.43},
310: {1.0: 9.41, 1.25: 11.22, 1.50: 11.03, 1.75: 11.60, 2.0: 13.39}}
# Apply lut to BT and sza values
def find_nearest_lut_sza(sza):
""" Get nearest look up table key value for given ssec(sza)"""
sza_opt = [1.0, 1.25, 1.50, 1.75, 2.0]
sza_idx = np.array([np.abs(sza - i) for i in sza_opt]).argmin()
return(sza_opt[sza_idx])
def find_nearest_lut_bt(bt):
""" Get nearest look up table key value for given BT"""
bt_opt = [260, 270, 280, 290, 300, 310]
bt_idx = np.array([np.abs(bt - i) for i in bt_opt]).argmin()
return(bt_opt[bt_idx])
def apply_lut(sza, bt):
""" Apply LUT to given BT and sza values"""
return(lut[bt][sza])
# Vectorize LUT functions for numpy arrays
vfind_nearest_lut_sza = np.vectorize(find_nearest_lut_sza)
vfind_nearest_lut_bt = np.vectorize(find_nearest_lut_bt)
vapply_lut = np.vectorize(apply_lut)
secsza_lut = vfind_nearest_lut_sza(secsza)
chn108_ma_lut = vfind_nearest_lut_bt(chn108)
bt_thres = vapply_lut(secsza_lut, chn108_ma_lut)
logger.debug("Set BT difference threshold for thin cirrus from %s to %s K"
% (np.min(bt_thres), np.max(bt_thres)))
# Create thin cirrus mask
bt_ci_mask = bt_diff > bt_thres
# Other cirrus test (T8.7–T10.8), founded on the relatively strong cirrus
# signal at the former wavelength (Wiegner et al.1998). Where the
# difference is greater than 0 K, cirrus is assumed to be present.
strong_ci_diff = chn87 - chn108
strong_ci_mask = strong_ci_diff > 0
cirrus_mask = bt_ci_mask | strong_ci_mask
fog_mask = fog_mask | cirrus_mask
filters['cirrus'] = np.nansum(fog_mask) - prev
prev += filters['cirrus']
# Those pixels whose cloud phase still remains undefined after these ice
# cloud exclusions are subjected to a much weaker cloud phase test in order
# to get an estimate regarding their phase. This test uses the NDSI
# introduced above. Where it falls below 0.1, a water cloud is assumed to
# be present.
chn16_ma = np.ma.masked_where(cloud_mask | snow_mask | ice_mask |
cirrus_mask, chn16)
chn06_ma = np.ma.masked_where(cloud_mask | snow_mask | ice_mask |
cirrus_mask, chn06)
ndsi_ci = (chn06 - chn16) / (chn06 + chn16)
water_mask = ndsi_ci > 0.1
fog_mask = fog_mask | water_mask
filters['water'] = np.nansum(fog_mask) - prev
prev += filters['water']
# Small droplet proxy test
# Fog generally has a stronger signal at 3.9 μm than clear ground, which
# in turn radiates more than other clouds.
# The 3.9 μm radiances for cloud-free land areas are averaged over 50 rows
# at a time to obtain an approximately latitudinal value.
# Wherever a cloud-covered pixel exceeds this value, it is flagged
# ‘small droplet cloud’.
cloud_free_ma = np.ma.masked_where(~cloud_mask, chn39)
chn39_ma = np.ma.masked_where(cloud_mask | snow_mask | ice_mask |
cirrus_mask | water_mask, chn39)
# Latitudinal average cloud free radiances
lat_cloudfree = np.ma.mean(cloud_free_ma, 1)
logger.debug("Mean latitudinal threshold for cloudfree areas: %.2f K"
% np.mean(lat_cloudfree))
global line
line = 0
def find_watercloud(lat, thres):
"""Funciton to compare row of BT with given latitudinal thresholds"""
global line
if all(lat.mask):
res = lat.mask
elif np.ma.is_masked(thres[line]):
res = lat <= np.mean(lat_cloudfree)
else:
res = lat <= thres[line]
line += 1
return(res)
# Apply latitudinal threshold to cloudy areas
drop_mask = np.apply_along_axis(find_watercloud, 1, chn39,
lat_cloudfree)
fog_mask = fog_mask | drop_mask
filters['drop'] = np.nansum(fog_mask) - prev
prev += filters['drop']
# Apply previous defined filters
chn108_ma = np.ma.masked_where(fog_mask, chn108)
logger.debug("Number of filtered non-fog pixels: %s"
% (np.sum(chn108_ma.mask)))
logger.debug("Number of potential fog cloud pixels: %s"
% (np.sum(~chn108_ma.mask)))
# 2. Spatial clustering
logger.info("### Analizing spatial properties of filtered fog clouds ###")
# Enumerate fog cloud clusters
cluster = measurements.label(~chn108_ma.mask)
# Get 10.8 channel sampled by the previous fog filters
cluster_ma = np.ma.masked_where(fog_mask, cluster[0])
# Get cloud and snow free cells
clear_ma = np.ma.masked_where(~cloud_mask | snow_mask, chn108)
logger.debug("Number of spatial coherent fog cloud clusters: %s"
% np.nanmax(np.unique(cluster_ma)))
# 3. Altitude test
def sliding_window(arr, window_size):
""" Construct a sliding window view of the array"""
arr = np.asarray(arr)
window_size = int(window_size)
if arr.ndim != 2:
raise ValueError("need 2-D input")
if not (window_size > 0):
raise ValueError("need a positive window size")
shape = (arr.shape[0] - window_size + 1,
arr.shape[1] - window_size + 1,
window_size, window_size)
if shape[0] <= 0:
shape = (1, shape[1], arr.shape[0], shape[3])
if shape[1] <= 0:
shape = (shape[0], 1, shape[2], arr.shape[1])
strides = (arr.shape[1]*arr.itemsize, arr.itemsize,
arr.shape[1]*arr.itemsize, arr.itemsize)
return as_strided(arr, shape=shape, strides=strides)
def cell_neighbors(arr, i, j, d, value):
"""Return d-th neighbors of cell (i, j)"""
w = sliding_window(arr, 2*d+1)
ix = np.clip(i - d, 0, w.shape[0]-1)
jx = np.clip(j - d, 0, w.shape[1]-1)
i0 = max(0, i - d - ix)
j0 = max(0, j - d - jx)
i1 = w.shape[2] - max(0, d - i + ix)
j1 = w.shape[3] - max(0, d - j + jx)
# Get cell value
if i1 - i0 == 3:
icell = 1
elif (i1 - i0 == 2) & (i0 == 0):
icell = 0
elif (i1 - i0 == 2) & (i0 == 1):
icell = 2
if j1 - j0 == 3:
jcell = 1
elif (j1 - j0 == 2) & (j0 == 0):
jcell = 0
elif (j1 - j0 == 2) & (j0 == 1):
jcell = 2
irange = range(i0, i1)
jrange = range(j0, j1)
neighbors = [w[ix, jx][k, l] for k in irange for l in jrange
if k != icell or l != jcell]
center = value[i, j] # Get center cell value from additional array
return(center, neighbors)
def get_fog_cth(cluster, cf_arr, bt_cc, elevation):
"""Get neighboring cloud free BT and elevation values of potenital
fog cloud clusters and compute cloud top height from naximum BT
differences for fog cloud contaminated pixel in comparison to cloud
free areas and their corresponding elevation
"""
e = 1
from collections import defaultdict
result = defaultdict(list)
elevation_ma = np.ma.masked_where(~cloud_mask | snow_mask, elevation)
# Convert masked values to nan
if np.ma.isMaskedArray(cf_arr):
cf_arr = cf_arr.filled(np.nan)
for index, val in np.ndenumerate(cluster):
if val != 0:
# Get list of cloud free neighbor pixel
tcc, tneigh = cell_neighbors(cf_arr, *index, d=1,
value=bt_cc)
zcc, zneigh = cell_neighbors(elevation_ma, *index, d=1,
value=elevation)
tcf_diff = np.array([tcf - tcc for tcf in tneigh])
zcf_diff = np.array([zcf - zcc for zcf in zneigh])
# Get maximum bt difference
try:
maxd = np.nanargmax(tcf_diff)
except ValueError:
continue
# compute cloud top height with constant athmospere temperature
# lapse rate
rate = 0.65
cth = tcf_diff[maxd] / rate * 100 - zcf_diff[maxd]
result[val].append(cth)
return(result)
# Calculate fog cluster cloud top height
cluster_h = get_fog_cth(cluster_ma, clear_ma, chn108_ma, elevation)
# Apply maximum threshold for cluster height to identify low fog clouds
cluster_mask = cluster_ma.mask
for key, item in cluster_h.iteritems():
if any([c > 2000 for c in item]):
cluster_mask[cluster_ma[key]] = True
# Create additional fog cluster map
cluster_cth = np.ma.masked_where(cluster_mask, cluster_ma)
for key, item in cluster_h.iteritems():
if all([c <= 2000 for c in item]):
cluster_cth[cluster_cth[key]] = np.mean(item)
# Update fog filter
fog_mask = fog_mask | cluster_mask
filters['height'] = np.nansum(fog_mask) - prev
prev += filters['height']
# Apply previous defined spatial filters
chn108_ma = np.ma.masked_where(cluster_mask, chn108)
# Surface homogenity test
cluster, nlbl = ndimage.label(~chn108_ma.mask)
cluster_ma = np.ma.masked_where(cluster_mask, cluster)
cluster_sd = ndimage.standard_deviation(chn108_ma, cluster_ma,
index=np.arange(1, nlbl+1))
# 4. Mask potential fog clouds with high spatial inhomogenity
sd_mask = cluster_sd > 2.5
cluster_dict = {key: sd_mask[key - 1] for key in np.arange(1, nlbl+1)}
for val in np.arange(1, nlbl+1):
cluster_mask[cluster_ma == val] = cluster_dict[val]
fog_mask = fog_mask | cluster_mask
filters['homogen'] = np.nansum(fog_mask) - prev
prev += filters['homogen']
# Apply previous defined spatial filters
chn108_ma = np.ma.masked_where(cluster_mask, chn108)
logger.debug("Number of spatial filtered non-fog pixels: %s"
% (np.sum(chn108_ma.mask)))
logger.debug("Number of remaining fog cloud pixels: %s"
% (np.sum(~chn108_ma.mask)))
# 5. Apply microphysical fog cloud filters
# Typical microphysical parameters for fog were taken from previous studies
# Fog optical depth normally ranges between 0.15 and 30 while droplet
# effective radius varies between 3 and 12 μm, with a maximum of 20 μm in
# coastal fog. The respective maxima for optical depth (30) and droplet
# radius (20 μm) are applied to the low stratus mask as cut-off levels.
# Where a pixel previously identified as fog/low stratus falls outside the
# range it will now be flagged as a non-fog pixel.
logger.info("### Apply microphysical plausible check ###")
if np.ma.isMaskedArray(cot):
cot = cot.base
if np.ma.isMaskedArray(reff):
reff = reff.base
# Add mask by microphysical thresholds
cpp_mask = cluster_mask | (cot > 30) | (reff > 20e-6)
fog_mask = fog_mask | cpp_mask
filters['cpp'] = np.nansum(fog_mask) - prev
prev += filters['cpp']
# Apply previous defined microphysical filters
chn108_ma = np.ma.masked_where(cpp_mask, chn108)
logger.debug("Number of microphysical filtered non-fog pixels: %s"
% (np.sum(chn108_ma.mask)))
logger.debug("Number of remaining fog cloud pixels: %s"
% (np.sum(~chn108_ma.mask)))
# Combine single masks to derive potential fog cloud filter
fls_mask = (fog_mask)
# Create debug output
filters['fls'] = np.nansum(fog_mask)
filters['remain'] = np.nansum(~fog_mask)
logger.info("""---- FLS algorithm filter results ---- \n
Number of initial pixels: {}
Removed non cloud pixel: {}
Removed snow pixels: {}
Removed ice cloud pixels: {}
Removed thin cirrus pixels: {}
Removed non water cloud pixels {}
Removed non small droplet pixels {}
Removed spatial high cloud pixels {}
Removed spatial inhomogen pixels {}
Removed microphysical tested pixels {}
---------------------------------------
Filtered non fog/low stratus pixels {}
Remaining fog/low stratus pixels {}
---------------------------------------
""".format(arrsize, filters['cloud'], filters['snow'],
filters['ice'], filters['cirrus'], filters['water'],
filters['drop'], filters['height'], filters['homogen'],
filters['cpp'], filters['fls'], filters['remain']))
return(fog_mask, cluster_h)
# Division factor to reduce image size
f = math.ceil(float(resolution / band_resolution_km))
# The projection x and y coordinates equals
# the scanning angle (in radians) multiplied by the satellite height (http://proj4.org/projections/geos.html)
X = file1.variables['x'][:][::f] * sat_h
Y = file1.variables['y'][:][::f] * sat_h
# map object with pyproj
p = Proj(proj='geos', h=sat_h, lon_0=sat_lon, sweep=sat_sweep, a=6378137.0)
# Convert map points to latitude and longitude with the magic provided by Pyproj
XX, YY = np.meshgrid(X, Y)
lons, lats = p(XX, YY, inverse=True)
# Get the year month day hour and minute to apply the zenith correction
utc_time = datetime(int(year), int(month), int(day), int(hour), int(minutes))
sun_zenith = np.zeros((data1.shape[0], data1.shape[1]))
sun_zenith = astronomy.sun_zenith_angle(utc_time, lons, lats)
print("Solar Zenith Angle calculus finished")
# Calculate the solar component (band 3.7 um)
from pyspectral.near_infrared_reflectance import Calculator
refl39 = Calculator('GOES-16', 'abi', 'ch7')
data1b = refl39.reflectance_from_tbs(sun_zenith, data1, data2)
print("Solar Component calculus finished")
#------------------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------------------
# RGB Components
R = data3
G = data4
B = data1b
def check_sunzen(self, config, area_def=None, xy_loc=None, lonlat=None,
data_name='local_data'):
'''Check if the data is within Sun zenith angle limits.
:param config: configuration options for this product
:type config: dict
:param area_def: area definition of the data
:type area_def: areadef
:param xy_loc: pixel location (2-tuple or 2-element list with x- and y-coordinates) where zenith angle limit is checked
:type xy_loc: tuple
:param lonlat: longitude/latitude location (2-tuple or 2-element list with longitude and latitude) where zenith angle limit is checked.
:type lonlat: tuple
:param data_name: name of the dataset to get data from
:type data_name: str
If both *xy_loc* and *lonlat* are None, image center is used as reference point. *xy_loc* overrides *lonlat*.
'''
LOGGER.info('Checking Sun zenith angle limits')
try:
data = getattr(self, data_name)
except AttributeError:
LOGGER.error('No such data: %s', data_name)
return False
if area_def is None and xy_loc is None:
LOGGER.error('No area definition or pixel location given')
return False
# Check availability of coordinates, load if necessary
if data.area.lons is None:
LOGGER.debug('Load coordinates for %s', data_name)
data.area.lons, data.area.lats = data.area.get_lonlats()
# Check availability of Sun zenith angles, calculate if necessary
try:
data.__getattribute__('sun_zen')
except AttributeError:
LOGGER.debug('Calculating Sun zenith angles for %s', data_name)
data.sun_zen = astronomy.sun_zenith_angle(data.time_slot,
data.area.lons,
data.area.lats)
if xy_loc is not None and len(xy_loc) == 2:
# Use the given xy-location
x_idx, y_idx = xy_loc
else:
if lonlat is not None and len(lonlat) == 2:
# Find the closest pixel to the given coordinates
dists = (data.area.lons - lonlat[0]) ** 2 + \
(data.area.lats - lonlat[1]) ** 2
y_idx, x_idx = np.where(dists == np.min(dists))
y_idx, x_idx = int(y_idx), int(x_idx)
else:
# Use image center
y_idx = int(area_def.y_size / 2)
x_idx = int(area_def.x_size / 2)
# Check if Sun is too low (day-only products)
try:
LOGGER.debug('Checking Sun zenith-angle limit at '
'(lon, lat) %3.1f, %3.1f (x, y: %d, %d)',
data.area.lons[y_idx, x_idx],
data.area.lats[y_idx, x_idx],
x_idx, y_idx)
if float(config['sunzen_day_maximum']) < \
data.sun_zen[y_idx, x_idx]:
LOGGER.info('Sun too low for day-time product.')
return False
except KeyError:
pass
# Check if Sun is too high (night-only products)
try:
if float(config['sunzen_night_minimum']) > \
data.sun_zen[y_idx, x_idx]:
LOGGER.info('Sun too high for night-time '
'product.')
return False
except KeyError:
pass
return True
