def __init__(self, filename, filename_info, filetype_info):
"""Initialize the reader."""
super(AHIHSDFileHandler, self).__init__(filename, filename_info,
filetype_info)
self.channels = dict([(i, None) for i in AHI_CHANNEL_NAMES])
self.units = dict([(i, 'counts') for i in AHI_CHANNEL_NAMES])
self._data = dict([(i, None) for i in AHI_CHANNEL_NAMES])
self._header = dict([(i, None) for i in AHI_CHANNEL_NAMES])
self.lons = None
self.lats = None
self.segment_number = filename_info['segment_number']
self.total_segments = filename_info['total_segments']
with open(self.filename) as fd:
self.basic_info = np.fromfile(fd,
dtype=_BASIC_INFO_TYPE,
count=1)
self.data_info = np.fromfile(fd,
dtype=_DATA_INFO_TYPE,
count=1)
self.proj_info = np.fromfile(fd,
dtype=_PROJ_INFO_TYPE,
count=1)[0]
self.nav_info = np.fromfile(fd,
dtype=_NAV_INFO_TYPE,
count=1)[0]
self.platform_name = np2str(self.basic_info['satellite'])
self.observation_area = np2str(self.basic_info['observation_area'])
self.sensor = 'ahi'
def __init__(self,
filename,
filename_info,
filetype_info,
mask_space=True,
calib_mode='update',
user_calibration=None,
round_actual_position=True):
"""Initialize the reader."""
super(AHIHSDFileHandler, self).__init__(filename, filename_info,
filetype_info)
self.is_zipped = False
self._unzipped = unzip_file(self.filename,
prefix=str(
filename_info['segment']).zfill(2))
# Assume file is not zipped
if self._unzipped:
# But if it is, set the filename to point to unzipped temp file
self.is_zipped = True
self.filename = self._unzipped
self.channels = dict([(i, None) for i in AHI_CHANNEL_NAMES])
self.units = dict([(i, 'counts') for i in AHI_CHANNEL_NAMES])
self._data = dict([(i, None) for i in AHI_CHANNEL_NAMES])
self._header = dict([(i, None) for i in AHI_CHANNEL_NAMES])
self.lons = None
self.lats = None
self.segment_number = filename_info['segment']
self.total_segments = filename_info['total_segments']
with open(self.filename) as fd:
self.basic_info = np.fromfile(fd, dtype=_BASIC_INFO_TYPE, count=1)
self.data_info = np.fromfile(fd, dtype=_DATA_INFO_TYPE, count=1)
self.proj_info = np.fromfile(fd, dtype=_PROJ_INFO_TYPE, count=1)[0]
self.nav_info = np.fromfile(fd, dtype=_NAV_INFO_TYPE, count=1)[0]
self.platform_name = np2str(self.basic_info['satellite'])
self.observation_area = np2str(self.basic_info['observation_area'])
self.sensor = 'ahi'
self.mask_space = mask_space
self.band_name = filetype_info['file_type'][4:].upper()
calib_mode_choices = ('NOMINAL', 'UPDATE')
if calib_mode.upper() not in calib_mode_choices:
raise ValueError(
'Invalid calibration mode: {}. Choose one of {}'.format(
calib_mode, calib_mode_choices))
self.calib_mode = calib_mode.upper()
self.user_calibration = user_calibration
self._round_actual_position = round_actual_position
def __init__(self, filename, filename_info, filetype_info):
"""Initialize the reader."""
super(AHIHSDFileHandler, self).__init__(filename, filename_info,
filetype_info)
self.channels = dict([(i, None) for i in AHI_CHANNEL_NAMES])
self.units = dict([(i, 'counts') for i in AHI_CHANNEL_NAMES])
self._data = dict([(i, None) for i in AHI_CHANNEL_NAMES])
self._header = dict([(i, None) for i in AHI_CHANNEL_NAMES])
self.lons = None
self.lats = None
self.segment_number = filename_info['segment_number']
self.total_segments = filename_info['total_segments']
with open(self.filename) as fd:
self.basic_info = np.fromfile(fd,
dtype=_BASIC_INFO_TYPE,
count=1)
self.data_info = np.fromfile(fd,
dtype=_DATA_INFO_TYPE,
count=1)
self.proj_info = np.fromfile(fd,
dtype=_PROJ_INFO_TYPE,
count=1)[0]
self.nav_info = np.fromfile(fd,
dtype=_NAV_INFO_TYPE,
count=1)[0]
self.platform_name = np2str(self.basic_info['satellite'])
self.sensor = 'ahi'
def _get_attr_value(self, obj, key):
value = getattr(obj, key)
try:
value = np2str(value)
except ValueError:
pass
return value
def _collect_attrs(self, name, attrs):
for key, value in six.iteritems(attrs):
value = np.squeeze(value)
fc_key = "{}/attr/{}".format(name, key)
try:
self.file_content[fc_key] = np2str(value)
except ValueError:
self.file_content[fc_key] = value
def _collect_attrs(self, name, obj):
"""Collect all the attributes for the provided file object."""
for key in obj.ncattrs():
value = getattr(obj, key)
fc_key = "{}/attr/{}".format(name, key)
try:
self.file_content[fc_key] = np2str(value)
except ValueError:
self.file_content[fc_key] = value
def _collect_attrs(self, name, obj):
"""Collect all the attributes for the provided file object.
"""
for key in obj.ncattrs():
value = getattr(obj, key)
fc_key = "{}/attr/{}".format(name, key)
try:
self.file_content[fc_key] = np2str(value)
except ValueError:
self.file_content[fc_key] = value
def test_np2str(self):
"""Test the np2str function."""
# byte object
npstring = np.string_('hej')
self.assertEqual(hf.np2str(npstring), 'hej')
# single element numpy array
np_arr = np.array([npstring])
self.assertEqual(hf.np2str(np_arr), 'hej')
# scalar numpy array
np_arr = np.array(npstring)
self.assertEqual(hf.np2str(np_arr), 'hej')
# multi-element array
npstring = np.array([npstring, npstring])
self.assertRaises(ValueError, hf.np2str, npstring)
# non-array
self.assertRaises(ValueError, hf.np2str, 5)
def __init__(self, filename, filename_info, filetype_info,
mask_space=True, calib_mode='nominal'):
"""Initialize the reader."""
super(AHIHSDFileHandler, self).__init__(filename, filename_info,
filetype_info)
self.channels = dict([(i, None) for i in AHI_CHANNEL_NAMES])
self.units = dict([(i, 'counts') for i in AHI_CHANNEL_NAMES])
self._data = dict([(i, None) for i in AHI_CHANNEL_NAMES])
self._header = dict([(i, None) for i in AHI_CHANNEL_NAMES])
self.lons = None
self.lats = None
self.segment_number = filename_info['segment_number']
self.total_segments = filename_info['total_segments']
with open(self.filename) as fd:
self.basic_info = np.fromfile(fd,
dtype=_BASIC_INFO_TYPE,
count=1)
self.data_info = np.fromfile(fd,
dtype=_DATA_INFO_TYPE,
count=1)
self.proj_info = np.fromfile(fd,
dtype=_PROJ_INFO_TYPE,
count=1)[0]
self.nav_info = np.fromfile(fd,
dtype=_NAV_INFO_TYPE,
count=1)[0]
self.platform_name = np2str(self.basic_info['satellite'])
self.observation_area = np2str(self.basic_info['observation_area'])
self.sensor = 'ahi'
self.mask_space = mask_space
calib_mode_choices = ('NOMINAL', 'UPDATE')
if calib_mode.upper() not in calib_mode_choices:
raise ValueError('Invalid calibration mode: {}. Choose one of {}'.format(
calib_mode, calib_mode_choices))
self.calib_mode = calib_mode.upper()
def __init__(self, filename, filename_info, filetype_info):
super(VIIRSCompactFileHandler, self).__init__(filename, filename_info,
filetype_info)
self.h5f = h5py.File(self.filename, "r")
self.finfo = filename_info
self.lons = None
self.lats = None
self.scans = self.h5f["All_Data"]["NumberOfScans"][0]
for key in self.h5f["All_Data"].keys():
if key.startswith("VIIRS") and key.endswith("GEO_All"):
self.geostuff = self.h5f["All_Data"][key]
break
self.c_align = self.geostuff["AlignmentCoefficient"].value[
np.newaxis, np.newaxis, :, np.newaxis]
self.c_exp = self.geostuff["ExpansionCoefficient"].value[np.newaxis,
np.newaxis, :,
np.newaxis]
for key in self.h5f["All_Data"].keys():
if key.startswith("VIIRS") and key.endswith("SDR_All"):
channel = key.split('-')[1]
break
# FIXME: this supposes there is only one tiepoint zone in the
# track direction
self.scan_size = self.h5f["All_Data/VIIRS-%s-SDR_All" %
channel].attrs["TiePointZoneSizeTrack"]
self.scan_size = np.asscalar(self.scan_size)
self.track_offset = self.h5f["All_Data/VIIRS-%s-SDR_All" %
channel].attrs["PixelOffsetTrack"]
self.scan_offset = self.h5f["All_Data/VIIRS-%s-SDR_All" %
channel].attrs["PixelOffsetScan"]
try:
self.group_locations = self.geostuff[
"TiePointZoneGroupLocationScanCompact"].value
except KeyError:
self.group_locations = [0]
self.tpz_sizes = self.h5f["All_Data/VIIRS-%s-SDR_All" %
channel].attrs["TiePointZoneSizeScan"]
self.nb_tpzs = self.geostuff["NumberOfTiePointZonesScan"].value
self.cache = {}
self.mda = {}
short_name = np2str(self.h5f.attrs['Platform_Short_Name'])
self.mda['platform_name'] = short_names.get(short_name, short_name)
self.mda['sensor'] = 'viirs'
def _collect_attrs(self, name, attrs):
for key, value in six.iteritems(attrs):
value = np.squeeze(value)
fc_key = "{}/attr/{}".format(name, key)
try:
self.file_content[fc_key] = np2str(value)
except ValueError:
self.file_content[fc_key] = value
except AttributeError:
# A HDF5 reference ?
value = self.get_reference(name, key)
if value is None:
LOG.warning("Value cannot be converted - skip setting attribute %s", fc_key)
else:
self.file_content[fc_key] = value
def _collect_attrs(self, name, attrs):
attrs_cache = self._attrs_cache.setdefault(name, {})
for key, value in attrs.items():
value = np.squeeze(value)
fc_key = "{}/attr/{}".format(name, key)
try:
value = np2str(value)
except ValueError:
# use the original value
pass
except AttributeError:
# A HDF5 reference ?
value = self.get_reference(name, key)
if value is None:
LOG.warning("Value cannot be converted - skip setting attribute %s", fc_key)
continue
self.file_content[fc_key] = attrs_cache[key] = value
def __init__(self, filename, filename_info, filetype_info):
"""Initialize the reader."""
super(VIIRSCompactFileHandler, self).__init__(filename, filename_info,
filetype_info)
self.h5f = h5py.File(self.filename, "r")
self.finfo = filename_info
self.lons = None
self.lats = None
if filetype_info['file_type'] == 'compact_m':
self.ch_type = 'MOD'
elif filetype_info['file_type'] == 'compact_dnb':
self.ch_type = 'DNB'
else:
raise IOError('Compact Viirs file type not recognized.')
geo_data = self.h5f["Data_Products"]["VIIRS-%s-GEO" % self.ch_type]["VIIRS-%s-GEO_Gran_0" % self.ch_type]
self.min_lat = np.asscalar(geo_data.attrs['South_Bounding_Coordinate'])
self.max_lat = np.asscalar(geo_data.attrs['North_Bounding_Coordinate'])
self.min_lon = np.asscalar(geo_data.attrs['West_Bounding_Coordinate'])
self.max_lon = np.asscalar(geo_data.attrs['East_Bounding_Coordinate'])
self.switch_to_cart = ((abs(self.max_lon - self.min_lon) > 90)
or (max(abs(self.min_lat), abs(self.max_lat)) > 60))
self.scans = self.h5f["All_Data"]["NumberOfScans"][0]
self.geostuff = self.h5f["All_Data"]['VIIRS-%s-GEO_All' % self.ch_type]
self.c_align = da.from_array(self.geostuff["AlignmentCoefficient"])[
np.newaxis, np.newaxis, :, np.newaxis]
self.c_exp = da.from_array(self.geostuff["ExpansionCoefficient"])[
np.newaxis, np.newaxis, :, np.newaxis]
for key in self.h5f["All_Data"].keys():
if key.startswith("VIIRS") and key.endswith("SDR_All"):
channel = key.split('-')[1]
break
# FIXME: this supposes there is only one tiepoint zone in the
# track direction
self.scan_size = self.h5f["All_Data/VIIRS-%s-SDR_All" %
channel].attrs["TiePointZoneSizeTrack"]
self.scan_size = np.asscalar(self.scan_size)
self.track_offset = self.h5f["All_Data/VIIRS-%s-SDR_All" %
channel].attrs["PixelOffsetTrack"]
self.scan_offset = self.h5f["All_Data/VIIRS-%s-SDR_All" %
channel].attrs["PixelOffsetScan"]
try:
self.group_locations = self.geostuff[
"TiePointZoneGroupLocationScanCompact"].value
except KeyError:
self.group_locations = [0]
self.tpz_sizes = self.h5f["All_Data/VIIRS-%s-SDR_All" %
channel].attrs["TiePointZoneSizeScan"]
self.nb_tpzs = self.geostuff["NumberOfTiePointZonesScan"].value
self.cache = {}
self.mda = {}
short_name = np2str(self.h5f.attrs['Platform_Short_Name'])
self.mda['platform_name'] = short_names.get(short_name, short_name)
self.mda['sensor'] = 'viirs'
def __init__(self, filename, filename_info, filetype_info):
"""Initialize the reader."""
super(VIIRSCompactFileHandler, self).__init__(filename, filename_info,
filetype_info)
self.h5f = h5py.File(self.filename, "r")
self.finfo = filename_info
self.lons = None
self.lats = None
if filetype_info['file_type'] == 'compact_m':
self.ch_type = 'MOD'
elif filetype_info['file_type'] == 'compact_dnb':
self.ch_type = 'DNB'
else:
raise IOError('Compact Viirs file type not recognized.')
geo_data = self.h5f["Data_Products"]["VIIRS-%s-GEO" % self.ch_type]["VIIRS-%s-GEO_Gran_0" % self.ch_type]
self.min_lat = geo_data.attrs['South_Bounding_Coordinate'].item()
self.max_lat = geo_data.attrs['North_Bounding_Coordinate'].item()
self.min_lon = geo_data.attrs['West_Bounding_Coordinate'].item()
self.max_lon = geo_data.attrs['East_Bounding_Coordinate'].item()
self.switch_to_cart = ((abs(self.max_lon - self.min_lon) > 90)
or (max(abs(self.min_lat), abs(self.max_lat)) > 60))
self.scans = self.h5f["All_Data"]["NumberOfScans"][0]
self.geography = self.h5f["All_Data"]['VIIRS-%s-GEO_All' % self.ch_type]
for key in self.h5f["All_Data"].keys():
if key.startswith("VIIRS") and key.endswith("SDR_All"):
channel = key.split('-')[1]
break
# This supposes there is only one tiepoint zone in the track direction.
channel_path = f"All_Data/VIIRS-{channel}-SDR_All"
self.scan_size = self.h5f[channel_path].attrs["TiePointZoneSizeTrack"].item()
self.track_offset = self.h5f[channel_path].attrs["PixelOffsetTrack"][()]
self.scan_offset = self.h5f[channel_path].attrs["PixelOffsetScan"][()]
try:
self.group_locations = self.geography["TiePointZoneGroupLocationScanCompact"][()]
except KeyError:
self.group_locations = [0]
self.tpz_sizes = da.from_array(self.h5f[channel_path].attrs["TiePointZoneSizeScan"], chunks=1)
if len(self.tpz_sizes.shape) == 2:
if self.tpz_sizes.shape[1] != 1:
raise NotImplementedError("Can't handle 2 dimensional tiepoint zones.")
self.tpz_sizes = self.tpz_sizes.squeeze(1)
self.nb_tiepoint_zones = self.geography["NumberOfTiePointZonesScan"][()]
self.c_align = da.from_array(self.geography["AlignmentCoefficient"],
chunks=tuple(self.nb_tiepoint_zones))
self.c_exp = da.from_array(self.geography["ExpansionCoefficient"],
chunks=tuple(self.nb_tiepoint_zones))
self.nb_tiepoint_zones = da.from_array(self.nb_tiepoint_zones, chunks=1)
self._expansion_coefs = None
self.cache = {}
self.mda = {}
short_name = np2str(self.h5f.attrs['Platform_Short_Name'])
self.mda['platform_name'] = short_names.get(short_name, short_name)
self.mda['sensor'] = 'viirs'