def calibrate(self,
dataset_id,
pre_launch_coeffs=False,
calib_coeffs=None):
"""Calibrate the data."""
if calib_coeffs is None:
calib_coeffs = {}
units = {
'reflectance': '%',
'brightness_temperature': 'K',
'counts': '',
'radiance': 'W*m-2*sr-1*cm ?'
}
if dataset_id['name'] in ("3a", "3b") and self._is3b is None:
# Is it 3a or 3b:
line_chunks = get_aapp_chunks((self._data.shape[0], 2048))[0]
self._is3a = da.bitwise_and(
da.from_array(self._data['scnlinbit'], chunks=line_chunks),
3) == 0
self._is3b = da.bitwise_and(
da.from_array(self._data['scnlinbit'], chunks=line_chunks),
3) == 1
try:
vis_idx = ['1', '2', '3a'].index(dataset_id['name'])
ir_idx = None
except ValueError:
vis_idx = None
ir_idx = ['3b', '4', '5'].index(dataset_id['name'])
mask = True
if vis_idx is not None:
coeffs = calib_coeffs.get('ch' + dataset_id['name'])
if dataset_id['name'] == '3a':
mask = self._is3a[:, None]
ds = create_xarray(
_vis_calibrate(self._data,
vis_idx,
dataset_id['calibration'],
pre_launch_coeffs,
coeffs,
mask=mask))
else:
if dataset_id['name'] == '3b':
mask = self._is3b[:, None]
ds = create_xarray(
_ir_calibrate(self._header,
self._data,
ir_idx,
dataset_id['calibration'],
mask=mask))
ds.attrs['units'] = units[dataset_id['calibration']]
ds.attrs.update(dataset_id._asdict())
return ds
def check(self):
"""Check VIS channel quality and issue a warning if it's bad."""
use_with_caution = da.bitwise_and(self._mask, 2)
if use_with_caution.all():
warnings.warn(
'All pixels of the VIS channel are flagged as "use with '
'caution". Use datasets "quality_pixel_bitmask" and '
'"data_quality_bitmask" to find out why.')
def _set_keep(self,
time_keep=None,
freq_keep=None,
corrprod_keep=None,
weights_keep=None,
flags_keep=None):
"""Set time, frequency and/or correlation product selection masks.
Set the selection masks for those parameters that are present. Also
include weights and flags selections as options.
Parameters
----------
time_keep : array of bool, shape (*T*,), optional
Boolean selection mask with one entry per timestamp
freq_keep : array of bool, shape (*F*,), optional
Boolean selection mask with one entry per frequency channel
corrprod_keep : array of bool, shape (*B*,), optional
Boolean selection mask with one entry per correlation product
weights_keep : 'all' or string or sequence of strings, optional
Names of selected weight types (or 'all' for the lot)
flags_keep : 'all' or string or sequence of strings, optional
Names of selected flag types (or 'all' for the lot)
"""
DataSet._set_keep(self, time_keep, freq_keep, corrprod_keep,
weights_keep, flags_keep)
update_all = time_keep is not None or freq_keep is not None or corrprod_keep is not None
update_flags = update_all or flags_keep is not None
if not self.source.data:
self._vis = self._weights = self._flags = None
elif update_flags:
# Create first-stage index from dataset selectors. Note: use
# the member variables, not the parameters, because the parameters
# can be None to indicate no change
stage1 = (self._time_keep, self._freq_keep, self._corrprod_keep)
if update_all:
# Cache dask graphs for the data fields
self._vis = DaskLazyIndexer(self.source.data.vis, stage1)
self._weights = DaskLazyIndexer(self.source.data.weights,
stage1)
flag_transforms = []
if ~self._flags_select != 0:
# Copy so that the lambda isn't affected by future changes
select = self._flags_select.copy()
flag_transforms.append(
lambda flags: da.bitwise_and(select, flags))
flag_transforms.append(lambda flags: flags.view(np.bool_))
self._flags = DaskLazyIndexer(self.source.data.flags, stage1,
flag_transforms)
def _set_keep(self, time_keep=None, freq_keep=None, corrprod_keep=None,
weights_keep=None, flags_keep=None):
"""Set time, frequency and/or correlation product selection masks.
Set the selection masks for those parameters that are present. Also
include weights and flags selections as options.
Parameters
----------
time_keep : array of bool, shape (*T*,), optional
Boolean selection mask with one entry per timestamp
freq_keep : array of bool, shape (*F*,), optional
Boolean selection mask with one entry per frequency channel
corrprod_keep : array of bool, shape (*B*,), optional
Boolean selection mask with one entry per correlation product
weights_keep : 'all' or string or sequence of strings, optional
Names of selected weight types (or 'all' for the lot)
flags_keep : 'all' or string or sequence of strings, optional
Names of selected flag types (or 'all' for the lot)
"""
DataSet._set_keep(self, time_keep, freq_keep, corrprod_keep, weights_keep, flags_keep)
update_all = time_keep is not None or freq_keep is not None or corrprod_keep is not None
update_flags = update_all or flags_keep is not None
if not self.source.data:
self._vis = self._weights = self._flags = self._excision = None
elif update_flags:
# Create first-stage index from dataset selectors. Note: use
# the member variables, not the parameters, because the parameters
# can be None to indicate no change
stage1 = (self._time_keep, self._freq_keep, self._corrprod_keep)
if update_all:
# Cache dask graphs for the data fields
self._vis = DaskLazyIndexer(self._corrected.vis, stage1)
self._weights = DaskLazyIndexer(self._corrected.weights, stage1)
flag_transforms = []
if ~self._flags_select != 0:
# Copy so that the lambda isn't affected by future changes
select = self._flags_select.copy()
flag_transforms.append(lambda flags: da.bitwise_and(select, flags))
flag_transforms.append(lambda flags: flags.view(np.bool_))
self._flags = DaskLazyIndexer(self._corrected.flags, stage1, flag_transforms)
unscaled_weights = self._corrected.unscaled_weights
if unscaled_weights is None or self.accumulations_per_dump is None:
self._excision = None
else:
# The maximum / expected number of CBF dumps per SDP dump
cbf_dumps_per_sdp_dump = round(self.dump_period / self.cbf_dump_period)
accs_per_sdp_dump = np.float32(self.accumulations_per_dump)
accs_per_cbf_dump = accs_per_sdp_dump / np.float32(cbf_dumps_per_sdp_dump)
# Each unscaled weight represents the actual number of accumulations per SDP dump.
# Correct most of the weight compression artefacts by forcing each weight to be
# an integer multiple of CBF n_accs, and then convert it to an excision fraction.
def integer_cbf_dumps(w):
return da.round(w / accs_per_cbf_dump) * accs_per_cbf_dump
def excision_fraction(w):
return (accs_per_sdp_dump - w) / accs_per_sdp_dump
excision_transforms = [integer_cbf_dumps, excision_fraction]
self._excision = DaskLazyIndexer(unscaled_weights, stage1, excision_transforms)
def calibrate(self,
dataset_id,
pre_launch_coeffs=False,
calib_coeffs=None):
"""Calibrate the data."""
if calib_coeffs is None:
calib_coeffs = {}
units = {
'reflectance': '%',
'brightness_temperature': 'K',
'counts': '',
'radiance': 'W*m-2*sr-1*cm ?'
}
if dataset_id.name in ("3a", "3b") and self._is3b is None:
# Is it 3a or 3b:
self._is3a = da.bitwise_and(
da.from_array(self._data['scnlinbit'], chunks=LINE_CHUNK),
3) == 0
self._is3b = da.bitwise_and(
da.from_array(self._data['scnlinbit'], chunks=LINE_CHUNK),
3) == 1
if dataset_id.name == '3a' and not np.any(self._is3a):
raise ValueError("Empty dataset for channel 3A")
if dataset_id.name == '3b' and not np.any(self._is3b):
raise ValueError("Empty dataset for channel 3B")
try:
vis_idx = ['1', '2', '3a'].index(dataset_id.name)
ir_idx = None
except ValueError:
vis_idx = None
ir_idx = ['3b', '4', '5'].index(dataset_id.name)
mask = True
if vis_idx is not None:
coeffs = calib_coeffs.get('ch' + dataset_id.name)
if dataset_id.name == '3a':
mask = self._is3a[:, None]
ds = create_xarray(
_vis_calibrate(self._data,
vis_idx,
dataset_id.calibration,
pre_launch_coeffs,
coeffs,
mask=mask))
else:
if dataset_id.name == '3b':
mask = self._is3b[:, None]
ds = create_xarray(
_ir_calibrate(self._header,
self._data,
ir_idx,
dataset_id.calibration,
mask=mask))
ds.attrs['units'] = units[dataset_id.calibration]
ds.attrs.update(dataset_id._asdict())
return ds