def open_received(file_name):
"""
Given a file, try to find and return the appropriate reader object.
Parameters
----------
file_name : str
Returns
-------
CRSDTypeReader
Raises
------
SarpyIOError
"""
if not os.path.exists(file_name):
raise SarpyIOError('File {} does not exist.'.format(file_name))
# parse openers, if not already done
parse_openers()
# see if we can find a reader though trial and error
for opener in _openers:
reader = opener(file_name)
if reader is not None:
return reader
# If for loop completes, no matching file format was found.
raise SarpyIOError('Unable to determine received image format.')
def __init__(self, file_name):
"""
Parameters
----------
file_name : str
"""
if h5py is None:
raise ImportError(
"Can't read NISAR files, because the h5py dependency is missing."
)
if not os.path.isfile(file_name):
raise SarpyIOError('Path {} is not a file'.format(file_name))
with h5py.File(file_name, 'r') as hf:
# noinspection PyBroadException
try:
# noinspection PyUnusedLocal
gp = hf['/science/LSAR/SLC']
except:
raise SarpyIOError(
'The hdf5 file does not have required path /science/LSAR/SLC'
)
self._file_name = file_name
def __init__(self, file_object):
"""
Parameters
----------
file_object : str|BinaryIO
file name or file like object for a NITF 2.1 or 2.0 containing a SIDD
"""
self._img_headers = None
self._is_sidd = False
self._sidd_meta = None
self._sicd_meta = None
super(SIDDDetails, self).__init__(file_object)
if self._nitf_header.ImageSegments.subhead_sizes.size == 0:
raise SarpyIOError('There are no image segments defined.')
if self._nitf_header.GraphicsSegments.item_sizes.size > 0:
raise SarpyIOError('A SIDD file does not allow for graphics segments.')
if self._nitf_header.DataExtensions.subhead_sizes.size == 0:
raise SarpyIOError(
'A SIDD file requires at least one data extension, containing the '
'SIDD xml structure.')
# define the sidd and sicd metadata
self._find_sidd()
if not self.is_sidd:
raise SarpyIOError('Could not find SIDD xml data extensions.')
# populate the image details
self.img_segment_rows = numpy.zeros(self.img_segment_offsets.shape, dtype=numpy.int64)
self.img_segment_columns = numpy.zeros(self.img_segment_offsets.shape, dtype=numpy.int64)
for i, im_header in enumerate(self.img_headers):
self.img_segment_rows[i] = im_header.NROWS
self.img_segment_columns[i] = im_header.NCOLS
def __init__(self, file_name):
"""
Parameters
----------
file_name : str
"""
# verify that the file is a tiff file
self._tiff_details = TiffDetails(file_name)
# verify that ImageDescription tiff tag exists
if 'ImageDescription' not in self._tiff_details.tags:
raise SarpyIOError('No "ImageDescription" tag in the tiff.')
img_format = self._tiff_details.tags['ImageDescription']
# verify that ImageDescription has a reasonable format
try:
self._img_desc_tags = json.loads(
img_format) # type: Dict[str, Any]
except Exception as e:
msg = 'Failed deserializing the ImageDescription tag as json with error {}'.format(
e)
logger.info(msg)
raise SarpyIOError(msg)
# verify the file is not compressed
self._tiff_details.check_compression()
# verify the file is not tiled
self._tiff_details.check_tiled()
def __init__(self, file_name):
self._file_name = file_name
self._user_data = None
self._data_offset = 20
self._caspr_data = None
self._symmetry = (False, False, False)
self._sicd = None
if not os.path.isfile(file_name):
raise SarpyIOError('Path {} is not a file'.format(file_name))
with open(file_name, 'rb') as fi:
self._magic_number = struct.unpack('I', fi.read(4))[0]
endian = self.ENDIAN.get(self._magic_number, None)
if endian is None:
raise SarpyIOError(
'File {} is not an SIO file. Got magic number {}'.format(
file_name, self._magic_number))
# reader basic header - (rows, columns, data_type, pixel_size)?
init_head = numpy.array(struct.unpack('{}4I'.format(endian),
fi.read(16)),
dtype=numpy.uint64)
if not (numpy.all(init_head[2:] == numpy.array([13, 8]))
or numpy.all(init_head[2:] == numpy.array([12, 4]))
or numpy.all(init_head[2:] == numpy.array([11, 2]))):
raise SarpyIOError(
'Got unsupported sio data type/pixel size = {}'.format(
init_head[2:]))
self._head = init_head
def __init__(self, file_name):
"""
Parameters
----------
file_name : str
"""
if h5py is None:
raise ImportError(
"Can't read ICEYE files, because the h5py dependency is missing."
)
if not os.path.isfile(file_name):
raise SarpyIOError('Path {} is not a file'.format(file_name))
with h5py.File(file_name, 'r') as hf:
if 's_q' not in hf or 's_i' not in hf:
raise SarpyIOError(
'The hdf file does not have the real (s_q) or imaginary dataset (s_i).'
)
if 'satellite_name' not in hf:
raise SarpyIOError(
'The hdf file does not have the satellite_name dataset.')
if 'product_name' not in hf:
raise SarpyIOError(
'The hdf file does not have the product_name dataset.')
self._file_name = file_name
def __init__(self, file_object: Union[str, BinaryIO]):
"""
Parameters
----------
file_object : str|BinaryIO
file name or file like object for a NITF 2.1 or 2.0 containing a SICD.
"""
self._des_index = None
self._des_header = None
self._img_headers = None
self._is_sicd = False
self._sicd_meta = None
NITFDetails.__init__(self, file_object)
if self._nitf_header.ImageSegments.subhead_sizes.size == 0:
raise SarpyIOError('There are no image segments defined.')
if self._nitf_header.GraphicsSegments.item_sizes.size > 0:
raise SarpyIOError('A SICD file does not allow for graphics segments.')
if self._nitf_header.DataExtensions.subhead_sizes.size == 0:
raise SarpyIOError(
'A SICD file requires at least one data extension, containing the '
'SICD xml structure.')
# define the sicd metadata
self._find_sicd()
if not self.is_sicd:
raise SarpyIOError('Could not find the SICD XML des.')
def open_complex(file_name: Union[str, BinaryIO]) -> SICDTypeReader:
"""
Given a file, try to find and return the appropriate reader object.
Parameters
----------
file_name : str|BinaryIO
Returns
-------
SICDTypeReader
Raises
------
SarpyIOError
"""
if (not is_file_like(file_name)) and (not os.path.exists(file_name)):
raise SarpyIOError('File {} does not exist.'.format(file_name))
# parse openers, if not already done
parse_openers()
# see if we can find a reader though trial and error
for opener in _openers:
reader = opener(file_name)
if reader is not None:
return reader
# check the final attempt openers
for opener in _define_final_attempt_openers():
reader = opener(file_name)
if reader is not None:
return reader
# If for loop completes, no matching file format was found.
raise SarpyIOError('Unable to determine complex image format.')
def __init__(self,
file_name: str,
reverse_axes: Union[None, int, Sequence[int]] = None,
transpose_axes: Optional[Tuple[int, ...]] = None):
"""
Parameters
----------
file_name : str
file name for a NITF file containing a complex SICD
reverse_axes : None|Sequence[int]
Any entries should be restricted to `{0, 1}`. The presence of
`0` means to reverse the rows (in the raw sense), and the presence
of `1` means to reverse the columns (in the raw sense).
transpose_axes : None|Tuple[int, ...]
If presented this should be only `(1, 0)`.
"""
self._reverse_axes = reverse_axes
self._transpose_axes = transpose_axes
self._segment_status = None
self._sicd_meta = None
self._segment_bands = None
NITFDetails.__init__(self, file_name)
self._find_complex_image_segments()
if len(self.sicd_meta) == 0:
raise SarpyIOError(
'No complex valued image segments found in file {}'.format(
file_name))
def _validate_filename(output_directory, output_file, sidd_structure):
"""
Validate the output filename.
Parameters
----------
output_directory : str
output_file : None|str
sidd_structure
Returns
-------
str
"""
if output_file is None:
# noinspection PyProtectedMember
fstem = os.path.split(sidd_structure.NITF['SUGGESTED_NAME'] +
'.nitf')[1]
else:
fstem = os.path.split(output_file)[1]
full_filename = os.path.join(os.path.expanduser(output_directory), fstem)
if os.path.exists(full_filename):
raise SarpyIOError('File {} already exists.'.format(full_filename))
return full_filename
def validate_filename():
if output_file is None:
return
if check_existence and os.path.exists(output_file):
raise SarpyIOError(
'The file {} already exists.'.format(output_file))
def __init__(self, file_name, crsd_meta, check_existence=True):
"""
Parameters
----------
file_name : str
crsd_meta : CRSDType
check_existence : bool
Should we check if the given file already exists, and raises an exception if so?
"""
self._pvp_memmaps = None
self._support_memmaps = None
self._signal_memmaps = None
self._channel_map = None
self._support_map = None
self._writing_state = {
'header': False,
'pvp': {},
'support': {},
'signal': {}
}
self._closed = False
self._crsd_meta = crsd_meta
self._crsd_header = crsd_meta.make_file_header()
if check_existence and os.path.exists(file_name):
raise SarpyIOError(
'File {} already exists, and a new CRSD file can not be created '
'at this location'.format(file_name))
super(CRSDWriter1_0, self).__init__(file_name)
self._prepare_for_writing()
def __init__(self, file_object):
"""
Parameters
----------
file_object : str|BinaryIO
The path to or file like object referencing the CRSD file.
"""
self._crsd_version = None
self._crsd_header = None
self._crsd_meta = None
self._close_after = False
if isinstance(file_object, string_types):
if not os.path.exists(file_object) or not os.path.isfile(
file_object):
raise SarpyIOError(
'path {} does not exist or is not a file'.format(
file_object))
self._file_name = file_object
self._file_object = open(file_object, 'rb')
self._close_after = True
elif is_file_like(file_object):
self._file_object = file_object
if hasattr(file_object, 'name') and isinstance(
file_object.name, string_types):
self._file_name = file_object.name
else:
self._file_name = '<file like object>'
self._close_after = False
else:
raise TypeError('Got unsupported input type {}'.format(
type(file_object)))
self._file_object.seek(0, os.SEEK_SET)
head_bytes = self._file_object.read(10)
if not isinstance(head_bytes, bytes):
raise ValueError('Input file like object not open in bytes mode.')
if not head_bytes.startswith(b'CRSD'):
raise SarpyIOError(
'File {} does not appear to be a CRSD file.'.format(
self.file_name))
self._extract_version()
self._extract_header()
self._extract_crsd()
def __init__(self, file_name):
"""
Parameters
----------
file_name : str
"""
self._file_names = {}
self._features = None
self._head_feature = None
if not isinstance(file_name, string_types):
raise SarpyIOError(
'file_name is required to be a string path name.')
if not os.path.isfile(file_name):
raise SarpyIOError(
'file_name = {} is not a valid existing file'.format(
file_name))
root_dir, fname = os.path.split(file_name)
fstem, fext = os.path.splitext(fname)
if fext not in ['.ntf', '.nitf', '.json']:
raise SarpyIOError(
'file_name is expected to have extension .ntf, .nitf, .json')
if not (fstem.endswith('_C') or fstem.endswith('_M')
or fstem.endswith('_R') or fstem.endswith('_V')):
raise SarpyIOError(
'file_name is expected to follow naming pattern XXX_C.ntf, XXX_M.ntf, XXX_R.ntf, '
'or XXX_V.json')
fstem = fstem[:-2]
for part in ['C', 'M', 'R']:
for ext in ['.ntf', '.nitf']:
fil = os.path.join(root_dir,
'{}_{}{}'.format(fstem, part, ext))
if os.path.isfile(fil):
self._file_names[part] = fil
if part not in self._file_names:
raise SarpyIOError(
'Change detection file part {} not found'.format(part))
fil = os.path.join(root_dir, '{}_V.json'.format(fstem))
if os.path.isfile(fil):
self._file_names['V'] = fil
else:
logger.warning(
'Change detection file part V (i.e. json metadata) not found.')
self._set_features()
def close(self):
if self._closed:
return
fully_written = self._check_fully_written()
self._closed = True
if not fully_written:
raise SarpyIOError('CRSD file {} is not fully written'.format(
self._file_name))
def find_geoid_file_from_dir(dir_name, search_files=None):
"""
Find the geoid file.
Parameters
----------
dir_name : str
search_files : str|List[str]
Returns
-------
str
"""
geoid_dir = os.path.join(dir_name, 'geoid')
if not os.path.exists(geoid_dir):
raise SarpyIOError(
'Input is a directory, and beneath it we expect to find '
'files in directory "geoid"')
if search_files is None:
search_files = []
elif isinstance(search_files, string_types):
search_files = [
search_files,
]
else:
search_files = list(search_files)
for entry in _SEARCH_FILES:
if entry not in search_files:
search_files.append(entry)
our_file = None
for fil in search_files:
file_name = os.path.join(geoid_dir, fil)
if os.path.exists(file_name):
our_file = file_name
break
if our_file is None:
raise SarpyIOError(
'input is a directory and we expect to find one of the files {} '
'in the directory "geoid" beneath it'.format(search_files))
return our_file
def __init__(self, file_name):
"""
Parameters
----------
file_name : str
"""
if h5py is None:
raise ImportError(
"Can't read Cosmo Skymed files, because the h5py dependency is missing."
)
if not os.path.isfile(file_name):
raise SarpyIOError('Path {} is not a file'.format(file_name))
with h5py.File(file_name, 'r') as hf:
try:
self._mission_id = hf.attrs['Mission ID'].decode('utf-8')
except KeyError:
raise SarpyIOError(
'The hdf file does not have the top level attribute "Mission ID"'
)
try:
self._product_type = hf.attrs['Product Type'].decode('utf-8')
except KeyError:
raise SarpyIOError(
'The hdf file does not have the top level attribute "Product Type"'
)
if self._mission_id not in ['CSK', 'CSG', 'KMPS']:
raise ValueError(
'Expected hdf5 attribute `Mission ID` should be one of "CSK", "CSG", or "KMPS"). '
'Got Mission ID = {}.'.format(self._mission_id))
if 'SCS' not in self._product_type:
raise ValueError(
'Expected hdf to contain complex products '
'(attribute `Product Type` which contains "SCS"). '
'Got Product Type = {}'.format(self._product_type))
self._file_name = file_name
def base_reader(self, value):
if isinstance(value, str):
reader = None
# try to open as sicd type
try:
reader = open_complex(value)
except SarpyIOError:
pass
# try to open as phase_history
if reader is None:
try:
reader = open_phase_history(value)
except SarpyIOError:
pass
if reader is None:
try:
reader = open_received(value)
except SarpyIOError:
pass
if reader is None:
raise SarpyIOError('Could not open file {} as a one of the complex type readers'.format(value))
value = reader
elif isinstance(value, (tuple, list)):
value = AggregateComplexReader(value)
if not isinstance(value, AbstractReader):
raise TypeError('base_reader must be of type AbstractReader, got type {}'.format(type(value)))
if value.reader_type not in ["SICD", "CPHD", "CRSD"]:
raise SarpyIOError(
'base_reader.reader_type must be "SICD", "CPHD", or "CRSD", got {}'.format(value.reader_type))
self._base_reader = value
# noinspection PyProtectedMember
self._chippers = value._get_chippers_as_tuple()
self._index = 0
self._data_size = value.get_data_size_as_tuple()[0]
def __init__(self, file_name, symmetry=(False, False, False), split_bands=True):
"""
Parameters
----------
file_name : str
file name for a NITF file containing a complex SICD
symmetry : tuple
split_bands : bool
Split multiple complex bands into single bands?
"""
self._split_bands = split_bands
self._symmetry = symmetry
self._sicd_meta = None
self._complex_segments = None
super(ComplexNITFDetails, self).__init__(file_name)
if self._nitf_header.ImageSegments.subhead_sizes.size == 0:
raise SarpyIOError('There are no image segments defined.')
self._find_complex_image_segments()
if self._complex_segments is None:
raise SarpyIOError('No complex valued (I/Q) image segments found in file {}'.format(file_name))
def _validate_readers(readers):
"""
Validate the input reader/file collection.
Parameters
----------
readers : list|tuple
Returns
-------
Tuple[BaseReader]
"""
if not isinstance(readers, (list, tuple)):
raise TypeError(
'input argument must be a list or tuple of readers/files. Got type {}'
.format(type(readers)))
# get a reader for each entry, and make sure that they are sicd type
# validate each entry
the_readers = []
for i, entry in enumerate(readers):
if isinstance(entry, string_types):
try:
reader = open_complex(entry)
except SarpyIOError:
raise SarpyIOError(
'Attempted and failed to open {} (entry {} of the input argument) '
'using the complex opener.'.format(entry, i))
else:
reader = entry
if not isinstance(entry, BaseReader):
raise TypeError(
'All elements of the input argument must be file names or BaseReader instances. '
'Entry {} is of type {}'.format(i, type(entry)))
if reader.reader_type != "SICD":
raise ValueError(
'Entry {} of the input argument does not correspond to a sicd type reader. '
'Got type {}, with reader_type value {}'.format(
i, type(reader), reader.reader_type))
if not isinstance(reader, SICDTypeReader):
raise ValueError(
'Entry {} of the input argument does not correspond to a SICDTypeReader instance. '
'Got type {}.'.format(i, type(reader)))
the_readers.append(reader)
return tuple(the_readers)
def __init__(self, file_name):
self._file_name = file_name
with open(self._file_name, 'rb') as fi:
# NB: DTED is always big-endian
# the first 80 characters are header
# characters 80:728 are data set identification record
# characters 728:3428 are accuracy record
# the remainder is data records, but DTED is not quite a raster
header = struct.unpack('>80s', fi.read(80))[0].decode('utf-8')
if header[:3] != 'UHL':
raise SarpyIOError(
'File {} does not appear to be a DTED file.'.format(
self._file_name))
lon = float(header[4:7]) + float(header[7:9]) / 60. + float(
header[9:11]) / 3600.
lon = -lon if header[11] == 'W' else lon
lat = float(header[12:15]) + float(header[15:17]) / 60. + float(
header[17:19]) / 3600.
lat = -lat if header[19] == 'S' else lat
self._origin = numpy.array([lon, lat], dtype=numpy.float64)
self._spacing = numpy.array(
[float(header[20:24]), float(header[24:28])],
dtype=numpy.float64) / 36000.
self._shape = numpy.array(
[int(header[47:51]), int(header[51:55])], dtype=numpy.int64)
self._bounding_box = numpy.zeros((4, ), dtype=numpy.float64)
self._bounding_box[0::2] = self._origin
self._bounding_box[1::2] = self._origin + self._spacing * (
self._shape - 1)
# starting at 3428, the rest of the file is data records, but not quite a raster
# each "row" is a data record with 8 extra bytes at the beginning,
# and 4 extra (checksum) at the end - look to MIL-PRF-89020B for an explanation
# To enable memory map usage, we will spoof it as a raster and adjust column indices
shp = (int(self._shape[0]), int(self._shape[1]) + 6)
self._mem_map = numpy.memmap(self._file_name,
dtype=numpy.dtype('>i2'),
mode='r',
offset=3428,
shape=shp)
def base_reader(self, value):
if isinstance(value, str):
reader = None
try:
reader = open_received(value)
except SarpyIOError:
pass
if reader is None:
raise SarpyIOError('Could not open file {} as a CRSD reader'.format(value))
value = reader
if not isinstance(value, CRSDTypeReader):
raise TypeError('base_reader must be a CRSDReader, got type {}'.format(type(value)))
self._base_reader = value
# noinspection PyProtectedMember
self._chippers = value._get_chippers_as_tuple()
self._index = 0
self._data_size = value.get_data_size_as_tuple()[0]
def __init__(self, file_name):
"""
Parameters
----------
file_name : str
"""
if not (isinstance(file_name, string_types)
and os.path.isfile(file_name)):
raise SarpyIOError('Not a TIFF file.')
with open(file_name, 'rb') as fi:
# Try to read the basic tiff header
try:
fi_endian = fi.read(2).decode('utf-8')
except:
raise SarpyIOError('Not a TIFF file.')
if fi_endian == 'II':
self._endian = '<'
elif fi_endian == 'MM':
self._endian = '>'
else:
raise SarpyIOError(
'Invalid tiff endian string {}'.format(fi_endian))
# check the magic number
self._magic_number = numpy.fromfile(fi,
dtype='{}i2'.format(
self._endian),
count=1)[0]
if self._magic_number not in [42, 43]:
raise SarpyIOError(
'Not a valid tiff file, got magic number {}'.format(
self._magic_number))
if self._magic_number == 43:
rhead = numpy.fromfile(fi,
dtype='{}i2'.format(self._endian),
count=2)
if rhead[0] != 8:
raise SarpyIOError(
'Not a valid bigtiff. The offset size is given as {}'.
format(rhead[0]))
if rhead[1] != 0:
raise SarpyIOError(
'Not a valid bigtiff. The reserved entry of '
'the header is given as {} != 0'.format(rhead[1]))
self._file_name = file_name
self._tags = None
def base_reader(self, value):
if isinstance(value, str):
reader = None
try:
reader = open_complex(value)
except SarpyIOError:
pass
if reader is None:
raise SarpyIOError('Could not open file {} as a SICD type reader'.format(value))
value = reader
elif isinstance(value, (tuple, list)):
value = AggregateComplexReader(value)
if not isinstance(value, SICDTypeReader):
raise TypeError('base_reader must be a SICDTypeReader, got type {}'.format(type(value)))
self._base_reader = value
# noinspection PyProtectedMember
self._chippers = value._get_chippers_as_tuple()
self._index = 0
self._data_size = value.get_data_size_as_tuple()[0]
def _validate_readers(readers):
"""
Validate the input reader/file collection.
Parameters
----------
readers : Sequence[SICDTypeReader]
Returns
-------
Tuple[SICDTypeReader]
"""
if not isinstance(readers, Sequence):
raise TypeError('input argument must be a list or tuple of readers/files. Got type {}'.format(type(readers)))
# get a reader for each entry, and make sure that they are sicd type
# validate each entry
the_readers = []
for i, entry in enumerate(readers):
if isinstance(entry, str):
try:
reader = open_complex(entry)
except SarpyIOError:
raise SarpyIOError(
'Attempted and failed to open {} (entry {} of the input argument) '
'using the complex opener.'.format(entry, i))
else:
reader = entry
if not isinstance(reader, SICDTypeReader):
raise ValueError(
'Entry {} of the input argument does not correspond to a SICDTypeReader instance. '
'Got type {}.'.format(i, type(reader)))
the_readers.append(reader)
return tuple(the_readers)
def __init__(self, file_name):
"""
Parameters
----------
file_name : str
path to a egm2008 pgm file
"""
self._offset = None
self._scale = None
if os.path.isdir(file_name):
file_name = find_geoid_file_from_dir(file_name)
with open(file_name, "rb") as f:
line = f.readline()
if line != b"P5\012" and line != b"P5\015\012":
raise SarpyIOError("No PGM header")
headerlen = len(line)
while True:
line = f.readline().decode('utf-8')
if len(line) == 0:
raise SarpyIOError("EOF before end of file header")
headerlen += len(line)
if line.startswith('# Offset '):
try:
self._offset = int(line[9:])
except ValueError as e:
raise SarpyIOError("Error reading offset", e)
elif line.startswith('# Scale '):
try:
self._scale = float(line[8:])
except ValueError as e:
raise SarpyIOError("Error reading scale", e)
elif not line.startswith('#'):
try:
slin = line.split()
self._width, self._height = int(slin[0]), int(slin[1])
except ValueError as e:
raise SarpyIOError("Bad PGM width&height line", e)
break
line = f.readline().decode('utf-8')
headerlen += len(line)
levels = int(line)
if levels != 65535:
raise SarpyIOError("PGM file must have 65535 gray levels")
if self._offset is None:
raise SarpyIOError("PGM file does not contain offset")
if self._scale is None:
raise SarpyIOError("PGM file does not contain scale")
if self._width < 2 or self._height < 2:
raise SarpyIOError("Raster size too small")
self._header_length = headerlen
self._memory_map = numpy.memmap(file_name,
dtype=numpy.dtype('>u2'),
mode='r',
offset=self._header_length,
shape=(self._height, self._width))
self._lon_res = self._width/360.0
self._lat_res = (self._height - 1)/180.0
def __init__(self, file_name, sicd_meta, user_data=None, check_older_version=False, check_existence=True):
"""
Parameters
----------
file_name : str
sicd_meta : SICDType
user_data : None|Dict[str, str]
check_older_version : bool
Try to use an older version (1.1) of the SICD standard, for possible
application compliance issues?
check_existence : bool
Should we check if the given file already exists, and raises an exception if so?
"""
if check_existence and os.path.exists(file_name):
raise SarpyIOError('Given file {} already exists, and a new SIO file cannot be created here.'.format(file_name))
# choose magic number (with user data) and corresponding endian-ness
magic_number = 0xFD7F02FF
endian = SIODetails.ENDIAN[magic_number]
# define basic image details
image_size = (sicd_meta.ImageData.NumRows, sicd_meta.ImageData.NumCols)
pixel_type = sicd_meta.ImageData.PixelType
if pixel_type == 'RE32F_IM32F':
data_type = numpy.dtype('{}f4'.format(endian))
element_type = 13
element_size = 8
transform_data = 'COMPLEX'
elif pixel_type == 'RE16I_IM16I':
data_type = numpy.dtype('{}i2'.format(endian))
element_type = 12
element_size = 4
transform_data = complex_to_int
else:
data_type = numpy.dtype('{}u1'.format(endian))
element_type = 11
element_size = 2
transform_data = complex_to_amp_phase(sicd_meta.ImageData.AmpTable)
# construct the sio header
header = numpy.array(
[magic_number, image_size[0], image_size[1], element_type, element_size],
dtype='>u4')
# construct the user data - must be {str : str}
if user_data is None:
user_data = {}
uh_args = sicd_meta.get_des_details(check_older_version)
user_data['SICDMETA'] = sicd_meta.to_xml_string(tag='SICD', urn=uh_args['DESSHTN'])
data_offset = 20
with open(file_name, 'wb') as fi:
fi.write(struct.pack('{}5I'.format(endian), *header))
# write the user data - name size, name, value size, value
for name in user_data:
name_bytes = name.encode('utf-8')
fi.write(struct.pack('{}I'.format(endian), len(name_bytes)))
fi.write(struct.pack('{}{}s'.format(endian, len(name_bytes)), name_bytes))
val_bytes = user_data[name].encode('utf-8')
fi.write(struct.pack('{}I'.format(endian), len(val_bytes)))
fi.write(struct.pack('{}{}s'.format(endian, len(val_bytes)), val_bytes))
data_offset += 4 + len(name_bytes) + 4 + len(val_bytes)
# initialize the bip writer - we're ready to go
output_bands = 2
super(SIOWriter, self).__init__(file_name, image_size, data_type, output_bands,
transform_data=transform_data, data_offset=data_offset)
def create_detected_image_sidd(ortho_helper,
output_directory,
output_file=None,
block_size=10,
dimension=0,
bounds=None,
version=2,
include_sicd=True,
remap_function=None):
"""
Create a SIDD version of a basic detected image from a SICD type reader.
Parameters
----------
ortho_helper : OrthorectificationHelper
The ortho-rectification helper object.
output_directory : str
The output directory for the given file.
output_file : None|str
The file name, this will default to a sensible value.
block_size : int
The approximate processing block size to fetch, given in MB. The
minimum value for use here will be 1.
dimension : int
Which dimension to split over in block processing? Must be either 0 or 1.
bounds : None|numpy.ndarray|list|tuple
The sicd pixel bounds of the form `(min row, max row, min col, max col)`.
This will default to the full image.
version : int
The SIDD version to use, must be one of 1 or 2.
include_sicd : bool
Include the SICD structure in the SIDD file?
remap_function : None|MonochromaticRemap
The applied remap function. If one is not provided, then a default is
used. Required global parameters will be calculated if they are missing,
so the internal state of this remap function may be modified.
Returns
-------
None
Examples
--------
.. code-block:: python
import os
from sarpy.io.complex.converter import open_complex
from sarpy.processing.ortho_rectify import BivariateSplineMethod, NearestNeighborMethod, PGProjection
from sarpy.io.product.sidd_product_creation import create_detected_image_sidd
reader = open_complex('<sicd type object file name>')
ortho_helper = NearestNeighborMethod(reader, index=0)
# create a sidd version 2 file for the whole file
create_detected_image_sidd(ortho_helper, '<output directory>', block_size=10, version=2)
"""
if not os.path.isdir(output_directory):
raise SarpyIOError(
'output_directory {} does not exist or is not a directory'.format(
output_directory))
if not isinstance(ortho_helper, OrthorectificationHelper):
raise TypeError(
'ortho_helper is required to be an instance of OrthorectificationHelper, '
'got type {}'.format(type(ortho_helper)))
if remap_function is None:
remap_function = DEFAULT_IMG_REMAP(override_name='IMG_DEFAULT')
_validate_remap_function(remap_function)
# construct the ortho-rectification iterator - for a basic data fetcher
calculator = FullResolutionFetcher(ortho_helper.reader,
dimension=dimension,
index=ortho_helper.index,
block_size=block_size)
ortho_iterator = OrthorectificationIterator(ortho_helper,
calculator=calculator,
bounds=bounds,
remap_function=remap_function,
recalc_remap_globals=False)
# create the sidd structure
ortho_bounds = ortho_iterator.ortho_bounds
sidd_structure = create_sidd_structure(ortho_helper,
ortho_bounds,
product_class='Detected Image',
pixel_type='MONO{}I'.format(
remap_function.bit_depth),
version=version)
# set suggested name
sidd_structure.NITF[
'SUGGESTED_NAME'] = ortho_helper.sicd.get_suggested_name(
ortho_helper.index) + '_IMG'
# create the sidd writer
full_filename = _validate_filename(output_directory, output_file,
sidd_structure)
writer = SIDDWriter(full_filename, sidd_structure,
ortho_helper.sicd if include_sicd else None)
# iterate and write
for data, start_indices in ortho_iterator:
writer(data, start_indices=start_indices, index=0)
def create_dynamic_image_sidd(ortho_helper,
output_directory,
output_file=None,
dimension=0,
block_size=10,
bounds=None,
frame_count=9,
aperture_fraction=0.2,
method='FULL',
version=2,
include_sicd=True,
remap_function=None):
"""
Create a SIDD version of a Dynamic Image (Sub-Aperture Stack) from a SICD type reader.
Parameters
----------
ortho_helper : OrthorectificationHelper
The ortho-rectification helper object.
output_directory : str
The output directory for the given file.
output_file : None|str
The file name, this will default to a sensible value.
dimension : int
The dimension over which to split the sub-aperture.
block_size : int
The approximate processing block size to fetch, given in MB. The
minimum value for use here will be 1.
bounds : None|numpy.ndarray|list|tuple
The sicd pixel bounds of the form `(min row, max row, min col, max col)`.
This will default to the full image.
frame_count : int
The number of frames to calculate.
aperture_fraction : float
The relative size of each aperture window.
method : str
The subaperture processing method, which must be one of
`('NORMAL', 'FULL', 'MINIMAL')`.
version : int
The SIDD version to use, must be one of 1 or 2.
include_sicd : bool
Include the SICD structure in the SIDD file?
remap_function : None|MonochromaticRemap
The applied remap function. If one is not provided, then a default is
used. Required global parameters will be calculated if they are missing,
so the internal state of this remap function may be modified.
Returns
-------
None
Examples
--------
Create a basic dynamic image.
.. code-block:: python
import os
from sarpy.io.complex.converter import open_complex
from sarpy.io.product.sidd_product_creation import create_dynamic_image_sidd
from sarpy.processing.csi import CSICalculator
from sarpy.processing.ortho_rectify import NearestNeighborMethod
reader = open_complex('<sicd type object file name>')
ortho_helper = NearestNeighborMethod(reader, index=0)
create_dynamic_image_sidd(ortho_helper, '<output directory>', dimension=0, version=2)
"""
if not os.path.isdir(output_directory):
raise SarpyIOError(
'output_directory {} does not exist or is not a directory'.format(
output_directory))
if not isinstance(ortho_helper, OrthorectificationHelper):
raise TypeError(
'ortho_helper is required to be an instance of OrthorectificationHelper, '
'got type {}'.format(type(ortho_helper)))
# construct the subaperture calculator class
subap_calculator = SubapertureCalculator(
ortho_helper.reader,
dimension=dimension,
index=ortho_helper.index,
block_size=block_size,
frame_count=frame_count,
aperture_fraction=aperture_fraction,
method=method)
if remap_function is None:
remap_function = DEFAULT_DI_REMAP(override_name='DI_DEFAULT')
_validate_remap_function(remap_function)
# construct the ortho-rectification iterator
ortho_iterator = SubapertureOrthoIterator(ortho_helper,
calculator=subap_calculator,
bounds=bounds,
remap_function=remap_function,
recalc_remap_globals=False,
depth_first=True)
# create the sidd structure
ortho_bounds = ortho_iterator.ortho_bounds
sidd_structure = create_sidd_structure(ortho_helper,
ortho_bounds,
product_class='Dynamic Image',
pixel_type='MONO{}I'.format(
remap_function.bit_depth),
version=version)
# set suggested name
sidd_structure.NITF[
'SUGGESTED_NAME'] = subap_calculator.sicd.get_suggested_name(
subap_calculator.index) + '__DI'
the_sidds = []
for i in range(subap_calculator.frame_count):
this_sidd = sidd_structure.copy()
this_sidd.ProductCreation.ProductType = 'Frame {}'.format(i + 1)
the_sidds.append(this_sidd)
# create the sidd writer
if output_file is None:
# noinspection PyProtectedMember
full_filename = os.path.join(
output_directory, sidd_structure.NITF['SUGGESTED_NAME'] + '.nitf')
else:
full_filename = os.path.join(output_directory, output_file)
if os.path.exists(os.path.expanduser(full_filename)):
raise SarpyIOError('File {} already exists.'.format(full_filename))
writer = SIDDWriter(full_filename, the_sidds,
subap_calculator.sicd if include_sicd else None)
# iterate and write
for data, start_indices, the_frame in ortho_iterator:
writer(data, start_indices=start_indices, index=the_frame)
def create_csi_sidd(ortho_helper,
output_directory,
output_file=None,
dimension=0,
block_size=30,
bounds=None,
version=2,
include_sicd=True,
remap_function=None):
"""
Create a SIDD version of a Color Sub-Aperture Image from a SICD type reader.
Parameters
----------
ortho_helper : OrthorectificationHelper
The ortho-rectification helper object.
output_directory : str
The output directory for the given file.
output_file : None|str
The file name, this will default to a sensible value.
dimension : int
The dimension over which to split the sub-aperture.
block_size : int
The approximate processing block size to fetch, given in MB. The
minimum value for use here will be 1.
bounds : None|numpy.ndarray|list|tuple
The sicd pixel bounds of the form `(min row, max row, min col, max col)`.
This will default to the full image.
version : int
The SIDD version to use, must be one of 1 or 2.
include_sicd : bool
Include the SICD structure in the SIDD file?
remap_function : None|MonochromaticRemap
The applied remap function. For csi processing, this must explicitly be
an 8-bit remap. If one is not provided, then a default is used. Required
global parameters will be calculated if they are missing, so the internal
state of this remap function may be modified.
Returns
-------
None
Examples
--------
.. code-block:: python
import os
from sarpy.io.complex.converter import open_complex
from sarpy.io.product.sidd_product_creation import create_csi_sidd
from sarpy.processing.csi import CSICalculator
from sarpy.processing.ortho_rectify import NearestNeighborMethod
reader = open_complex('<sicd type object file name>')
ortho_helper = NearestNeighborMethod(reader, index=0)
create_csi_sidd(ortho_helper, '<output directory>', dimension=0, version=2)
"""
if not os.path.isdir(output_directory):
raise SarpyIOError(
'output_directory {} does not exist or is not a directory'.format(
output_directory))
if not isinstance(ortho_helper, OrthorectificationHelper):
raise TypeError(
'ortho_helper is required to be an instance of OrthorectificationHelper, '
'got type {}'.format(type(ortho_helper)))
# construct the CSI calculator class
csi_calculator = CSICalculator(ortho_helper.reader,
dimension=dimension,
index=ortho_helper.index,
block_size=block_size)
if remap_function is None:
remap_function = DEFAULT_CSI_REMAP(override_name='CSI_DEFAULT',
bit_depth=8)
_validate_remap_function(remap_function)
if remap_function.bit_depth != 8:
raise ValueError(
'The CSI SIDD specifically requires an 8-bit remap function.')
# construct the ortho-rectification iterator
ortho_iterator = OrthorectificationIterator(ortho_helper,
calculator=csi_calculator,
bounds=bounds,
remap_function=remap_function,
recalc_remap_globals=False)
# create the sidd structure
ortho_bounds = ortho_iterator.ortho_bounds
sidd_structure = create_sidd_structure(
ortho_helper,
ortho_bounds,
product_class='Color Subaperture Image',
pixel_type='RGB24I',
version=version)
# set suggested name
sidd_structure.NITF[
'SUGGESTED_NAME'] = csi_calculator.sicd.get_suggested_name(
csi_calculator.index) + '_CSI'
# create the sidd writer
full_filename = _validate_filename(output_directory, output_file,
sidd_structure)
writer = SIDDWriter(full_filename, sidd_structure,
csi_calculator.sicd if include_sicd else None)
# iterate and write
for data, start_indices in ortho_iterator:
writer(data, start_indices=start_indices, index=0)
