def __init__(self,
file_name: str,
reverse_axes: Union[None, int, Sequence[int]] = None,
transpose_axes: Optional[Tuple[int, ...]] = None):
"""
Parameters
----------
file_name : str
file name for a NITF file containing a complex SICD
reverse_axes : None|Sequence[int]
Any entries should be restricted to `{0, 1}`. The presence of
`0` means to reverse the rows (in the raw sense), and the presence
of `1` means to reverse the columns (in the raw sense).
transpose_axes : None|Tuple[int, ...]
If presented this should be only `(1, 0)`.
"""
self._reverse_axes = reverse_axes
self._transpose_axes = transpose_axes
self._segment_status = None
self._sicd_meta = None
self._segment_bands = None
NITFDetails.__init__(self, file_name)
self._find_complex_image_segments()
if len(self.sicd_meta) == 0:
raise SarpyIOError(
'No complex valued image segments found in file {}'.format(
file_name))
def __init__(self, file_object: Union[str, BinaryIO]):
"""
Parameters
----------
file_object : str|BinaryIO
file name or file like object for a NITF 2.1 or 2.0 containing a SICD.
"""
self._des_index = None
self._des_header = None
self._img_headers = None
self._is_sicd = False
self._sicd_meta = None
NITFDetails.__init__(self, file_object)
if self._nitf_header.ImageSegments.subhead_sizes.size == 0:
raise SarpyIOError('There are no image segments defined.')
if self._nitf_header.GraphicsSegments.item_sizes.size > 0:
raise SarpyIOError('A SICD file does not allow for graphics segments.')
if self._nitf_header.DataExtensions.subhead_sizes.size == 0:
raise SarpyIOError(
'A SICD file requires at least one data extension, containing the '
'SICD xml structure.')
# define the sicd metadata
self._find_sicd()
if not self.is_sicd:
raise SarpyIOError('Could not find the SICD XML des.')
def check_sicd_file(nitf_details):
"""
Check the validity of the given NITF file as a SICD file.
Parameters
----------
nitf_details : str|NITFDetails
The path to the NITF file, or a `NITFDetails` object.
Returns
-------
bool
"""
def check_data_extension_headers():
# type: () -> (str, Union[DataExtensionHeader, DataExtensionHeader0])
sicd_des = []
for i in range(nitf_details.des_subheader_offsets.size):
subhead_bytes = nitf_details.get_des_subheader_bytes(i)
des_bytes = None
if subhead_bytes.startswith(b'DEXML_DATA_CONTENT'):
des_bytes = nitf_details.get_des_bytes(i)
elif subhead_bytes.startswith(b'DESIDD_XML'):
raise ValueError(
'This file contains an old format SIDD DES, and should be a SIDD file')
elif subhead_bytes.startswith(b'DESICD_XML'):
des_bytes = nitf_details.get_des_bytes(i)
if des_bytes is None:
continue
# compare the SICD structure and the des header structure
if nitf_details.nitf_version == '02.00':
des_header = DataExtensionHeader0.from_bytes(subhead_bytes, start=0)
elif nitf_details.nitf_version == '02.10':
des_header = DataExtensionHeader.from_bytes(subhead_bytes, start=0)
else:
raise ValueError('Got unhandled NITF version {}'.format(nitf_details.nitf_version))
try:
des_string = des_bytes.decode('utf-8').strip()
root_node, xml_ns = parse_xml_from_string(des_string)
# namespace makes this ugly
if 'SIDD' in root_node.tag:
raise ValueError(
'This file contains a SIDD DES, and should be a SIDD file')
elif 'SICD' in root_node.tag:
sicd_des.append((i, des_string, des_header))
except Exception as e:
logger.error('Failed parsing the xml DES entry {} as xml'.format(i))
raise e
if len(sicd_des) == 0:
raise ValueError('No SICD DES values found, so this is not a viable SICD file')
elif len(sicd_des) > 1:
raise ValueError(
'Multiple SICD DES values found at indices {},\n'
'so this is not a viable SICD file'.format([entry[0] for entry in sicd_des]))
return sicd_des[0][1], sicd_des[0][2]
def check_image_data():
# type: () -> bool
# get pixel type
pixel_type = the_sicd.ImageData.PixelType
if pixel_type == 'RE32F_IM32F':
exp_nbpp = 32
exp_pvtype = 'R'
elif pixel_type == 'RE16I_IM16I':
exp_nbpp = 16
exp_pvtype = 'SI'
elif pixel_type == 'AMP8I_PHS8I':
exp_nbpp = 8
exp_pvtype = 'INT'
else:
raise ValueError('Got unexpected pixel type {}'.format(pixel_type))
valid_images = True
# verify that all images have the correct pixel type
for i, img_header in enumerate(nitf_details.img_headers):
if img_header.ICAT.strip() != 'SAR':
valid_images = False
logger.error(
'SICD: image segment at index {} of {} has ICAT = `{}`,\n\texpected to be `SAR`'.format(
i, len(nitf_details.img_headers), img_header.ICAT.strip()))
if img_header.PVTYPE.strip() != exp_pvtype:
valid_images = False
logger.error(
'SICD: image segment at index {} of {} has PVTYPE = `{}`,\n\t'
'expected to be `{}` based on pixel type {}'.format(
i, len(nitf_details.img_headers), img_header.PVTYPE.strip(), exp_pvtype, pixel_type))
if img_header.NBPP != exp_nbpp:
valid_images = False
logger.error(
'SICD: image segment at index {} of {} has NBPP = `{}`,\n\t'
'expected to be `{}` based on pixel type {}'.format(
i, len(nitf_details.img_headers), img_header.NBPP, exp_nbpp, pixel_type))
if len(img_header.Bands) != 2:
valid_images = False
logger.error('SICD: image segment at index {} of {} does not have two (I/Q or M/P) bands'.format(
i, len(nitf_details.img_headers)))
continue
if pixel_type == 'AMP8I_PHS8I':
if img_header.Bands[0].ISUBCAT.strip() != 'M' and img_header.Bands[1].ISUBCAT.strip() != 'P':
valid_images = False
logger.error(
'SICD: pixel_type is {}, image segment at index {} of {}\n\t'
'has bands with ISUBCAT {}, expected ("M", "P")'.format(
pixel_type, i, len(nitf_details.img_headers),
(img_header.Bands[0].ISUBCAT.strip(), img_header.Bands[1].ISUBCAT.strip())))
else:
if img_header.Bands[0].ISUBCAT.strip() != 'I' and img_header.Bands[1].ISUBCAT.strip() != 'Q':
valid_images = False
logger.error(
'SICD: pixel_type is {}, image segment at index {} of {}\n\t'
'has bands with ISUBCAT {}, expected ("I", "Q")'.format(
pixel_type, i, len(nitf_details.img_headers),
(img_header.Bands[0].ISUBCAT.strip(), img_header.Bands[1].ISUBCAT.strip())))
return valid_images
if isinstance(nitf_details, string_types):
if not os.path.isfile(nitf_details):
raise ValueError('Got string input, but it is not a valid path')
nitf_details = NITFDetails(nitf_details)
if not isinstance(nitf_details, NITFDetails):
raise TypeError(
'Input is expected to be a path to a NITF file, or a NITFDetails object instance')
# find the sicd header
sicd_xml_string, des_header = check_data_extension_headers()
# check that the sicd and header are valid
valid_sicd_des, the_sicd = check_sicd_data_extension(nitf_details, des_header, sicd_xml_string)
# check that the image segments all make sense compared to the sicd structure
valid_img = check_image_data()
all_valid = valid_sicd_des & valid_img
if valid_img:
try:
reader = SICDReader(nitf_details.file_name)
except Exception as e:
logger.exception(
'SICD: All image segments appear viable for the SICD,\n\t'
'but SICDReader construction failed')
return all_valid
def check_sidd_file(nitf_details):
"""
Check the validity of the given NITF file as a SICD file.
Parameters
----------
nitf_details : str|NITFDetails
Returns
-------
bool
"""
def find_des():
for i in range(nitf_details.des_subheader_offsets.size):
subhead_bytes = nitf_details.get_des_subheader_bytes(i)
if nitf_details.nitf_version == '02.00':
des_header = DataExtensionHeader0.from_bytes(subhead_bytes,
start=0)
elif nitf_details.nitf_version == '02.10':
des_header = DataExtensionHeader.from_bytes(subhead_bytes,
start=0)
else:
raise ValueError('Got unhandled NITF version {}'.format(
nitf_details.nitf_version))
if subhead_bytes.startswith(b'DEXML_DATA_CONTENT'):
des_bytes = nitf_details.get_des_bytes(i).decode(
'utf-8').strip().encode()
# noinspection PyBroadException
try:
root_node, xml_ns = parse_xml_from_string(des_bytes)
if 'SIDD' in root_node.tag: # namespace makes this ugly
sidd_des.append(
(i, des_bytes, root_node, xml_ns, des_header))
elif 'SICD' in root_node.tag:
sicd_des.append(
(i, des_bytes, root_node, xml_ns, des_header))
except Exception:
continue
elif subhead_bytes.startswith(b'DESIDD_XML'):
# This is an old format SIDD
des_bytes = nitf_details.get_des_bytes(i).decode(
'utf-8').strip().encode()
try:
root_node, xml_ns = parse_xml_from_string(des_bytes)
if 'SIDD' in root_node.tag: # namespace makes this ugly
sidd_des.append(
(i, des_bytes, root_node, xml_ns, des_header))
except Exception as e:
logger.exception(
'SIDD: Old-style SIDD DES header at index {}, but failed parsing'
.format(i))
continue
elif subhead_bytes.startswith(b'DESICD_XML'):
# This is an old format SICD
des_bytes = nitf_details.get_des_bytes(i).decode(
'utf-8').strip().encode()
try:
root_node, xml_ns = parse_xml_from_string(des_bytes)
if 'SICD' in root_node.tag: # namespace makes this ugly
sicd_des.append(
(i, des_bytes, root_node, xml_ns, des_header))
except Exception as e:
logger.exception(
'SIDD: Old-style SICD DES header at index {}, but failed parsing'
.format(i))
continue
def check_image_data():
valid_images = True
# verify that all images have the correct pixel type
for i, img_header in enumerate(nitf_details.img_headers):
if img_header.ICAT.strip() != 'SAR':
continue
iid1 = img_header.IID1.strip()
if re.match(r'^SIDD\d\d\d\d\d\d', iid1) is None:
valid_images = False
logger.error(
'SIDD: image segment at index {} of {} has IID1 = `{}`,\n\t'
'expected to be of the form `SIDDXXXYYY`'.format(
i, len(nitf_details.img_headers), iid1))
continue
sidd_index = int(iid1[4:7])
if not (0 < sidd_index <= len(sidd_des)):
valid_images = False
logger.error(
'SIDD: image segment at index {} of {} has IID1 = `{}`,\n\t'
'it is unclear with which of the {} SIDDs '
'this is associated'.format(i,
len(nitf_details.img_headers),
iid1, len(sidd_des)))
continue
has_image[sidd_index - 1] = True
type_information = sidd_nitf_details[sidd_index - 1]
pixel_type = type_information['pixel_type']
if pixel_type is None:
continue # we already noted the failure here
exp_nbpp, exp_pvtype = type_information['nbpp'], type_information[
'pvtype']
if img_header.PVTYPE.strip() != exp_pvtype:
valid_images = False
logger.error(
'SIDD: image segment at index {} of {} has PVTYPE = `{}`,\n\t'
'expected to be `{}` based on pixel type {}'.format(
i, len(nitf_details.img_headers),
img_header.PVTYPE.strip(), exp_pvtype, pixel_type))
if img_header.NBPP != exp_nbpp:
valid_images = False
logger.error(
'SIDD: image segment at index {} of {} has NBPP = `{}`,\n\t'
'expected to be `{}` based on pixel type {}'.format(
i, len(nitf_details.img_headers), img_header.NBPP,
exp_nbpp, pixel_type))
for sidd_index, entry in enumerate(has_image):
if not entry:
logger.error(
'SIDD: No image segments appear to be associated with the sidd at index {}'
.format(sidd_index))
valid_images = False
return valid_images
if isinstance(nitf_details, str):
if not os.path.isfile(nitf_details):
raise ValueError('Got string input, but it is not a valid path')
nitf_details = NITFDetails(nitf_details)
if not isinstance(nitf_details, NITFDetails):
raise TypeError(
'Input is expected to be a path to a NITF file, or a NITFDetails object instance'
)
sidd_des = []
sicd_des = []
sidd_nitf_details = []
has_image = []
find_des()
if len(sidd_des) < 1:
logger.error('SIDD: No SIDD DES found, this is not a valid SIDD file.')
return False
valid_sidd_des = True
for entry in sidd_des:
this_sidd_valid, the_sidd = check_sidd_data_extension(
nitf_details, entry[4], entry[1])
valid_sidd_des &= this_sidd_valid
has_image.append(False)
if the_sidd.Display is None or the_sidd.Display.PixelType is None:
valid_sidd_des = False
logger.error(
'SIDD: SIDD.Display.PixelType is not populated, and can not be compared to NITF image details'
)
sidd_nitf_details.append({'pixel_type': None})
elif the_sidd.Display.PixelType in ['MONO8I', 'MONO8LU', 'RGB8LU']:
sidd_nitf_details.append({
'nbpp': 8,
'pvtype': 'INT',
'pixel_type': the_sidd.Display.PixelType
})
elif the_sidd.Display.PixelType == 'MONO16I':
sidd_nitf_details.append({
'nbpp': 16,
'pvtype': 'INT',
'pixel_type': the_sidd.Display.PixelType
})
elif the_sidd.Display.PixelType == 'RGB24I':
sidd_nitf_details.append({
'nbpp': 24,
'pvtype': 'INT',
'pixel_type': the_sidd.Display.PixelType
})
else:
raise ValueError('Got unhandled pixel type {}'.format(
the_sidd.Display.PixelType))
valid_sicd_des = True
for entry in sicd_des:
this_sicd_valid, _ = check_sicd_data_extension(nitf_details, entry[4],
entry[1])
valid_sicd_des &= this_sicd_valid
valid_image = check_image_data()
return valid_sidd_des & valid_sicd_des & valid_image
def generic_nitf_header_test(instance, test_file):
assert isinstance(instance, unittest.TestCase)
# can we parse it at all? how long does it take?
with instance.subTest(msg="header parsing"):
start = time.time()
details = NITFDetails(test_file)
# how long does it take?
logging.info('unpacked nitf details in {}'.format(time.time() - start))
# how does it look?
logging.debug(details.nitf_header)
# is the output as long as it should be?
with instance.subTest(msg="header length match"):
header_string = details.nitf_header.to_bytes()
equality = (len(header_string) == details.nitf_header.HL)
if not equality:
logging.error(
'len(produced header) = {}, nitf_header.HL = {}'.format(
len(header_string), details.nitf_header.HL))
instance.assertTrue(equality)
# is the output what it should be?
with instance.subTest(msg="header content match"):
with open(test_file, 'rb') as fi:
file_header = fi.read(details.nitf_header.HL)
equality = (file_header == header_string)
if not equality:
chunk_size = 80
start_chunk = 0
while start_chunk < len(header_string):
end_chunk = min(start_chunk + chunk_size, len(header_string))
logging.error('real[{}:{}] = {}'.format(
start_chunk, end_chunk,
file_header[start_chunk:end_chunk]))
logging.error('prod[{}:{}] = {}'.format(
start_chunk, end_chunk,
header_string[start_chunk:end_chunk]))
start_chunk = end_chunk
instance.assertTrue(equality)
# is each image subheader working?
if details.img_segment_offsets is not None:
for i in range(details.img_segment_offsets.size):
with instance.subTest('image subheader {} match'.format(i)):
img_bytes = details.get_image_subheader_bytes(i)
img_sub = ImageSegmentHeader.from_bytes(img_bytes, start=0)
instance.assertEqual(len(img_bytes),
img_sub.get_bytes_length(),
msg='image subheader as long as expected')
instance.assertEqual(
img_bytes,
img_sub.to_bytes(),
msg=
'image subheader serializes and deserializes as expected')
# is each text segment working?
if details.text_segment_offsets is not None:
for i in range(details.text_segment_offsets.size):
with instance.subTest('text subheader {} match'.format(i)):
txt_bytes = details.get_text_subheader_bytes(i)
txt_sub = TextSegmentHeader.from_bytes(txt_bytes, start=0)
instance.assertEqual(len(txt_bytes),
txt_sub.get_bytes_length(),
msg='text subheader as long as expected')
instance.assertEqual(
txt_bytes,
txt_sub.to_bytes(),
msg='text subheader serializes and deserializes as expected'
)
# is each graphics segment working?
if details.graphics_segment_offsets is not None:
for i in range(details.graphics_segment_offsets.size):
with instance.subTest('graphics subheader {} match'.format(i)):
graphics_bytes = details.get_graphics_subheader_bytes(i)
graphics_sub = GraphicsSegmentHeader.from_bytes(graphics_bytes,
start=0)
instance.assertEqual(
len(graphics_bytes),
graphics_sub.get_bytes_length(),
msg='graphics subheader as long as expected')
instance.assertEqual(
graphics_bytes,
graphics_sub.to_bytes(),
msg=
'graphics subheader serializes and deserializes as expected'
)
# is each data extenson subheader working?
if details.des_segment_offsets is not None:
for i in range(details.des_segment_offsets.size):
with instance.subTest('des subheader {} match'.format(i)):
des_bytes = details.get_des_subheader_bytes(i)
des_sub = DataExtensionHeader.from_bytes(des_bytes, start=0)
instance.assertEqual(len(des_bytes),
des_sub.get_bytes_length(),
msg='des subheader as long as expected')
instance.assertEqual(
des_bytes,
des_sub.to_bytes(),
msg='des subheader serializes and deserializes as expected'
)
def print_nitf(file_name, dest=sys.stdout):
"""
Worker function to dump the NITF header and various subheader details to the
provided destination.
Parameters
----------
file_name : str|BinaryIO
dest : TextIO
"""
# Configure print function for desired destination
# - e.g., stdout, string buffer, file
global print_func
print_func = functools.partial(print, file=dest)
details = NITFDetails(file_name)
if isinstance(file_name, string_types):
print_func('')
print_func('Details for file {}'.format(file_name))
print_func('')
print_func('----- File Header -----')
_print_file_header(details.nitf_header)
print_func('')
if details.img_subheader_offsets is not None:
for img_subhead_num in range(details.img_subheader_offsets.size):
print_func('----- Image {} -----'.format(img_subhead_num))
hdr = details.parse_image_subheader(img_subhead_num)
_print_image_header(hdr)
print_func('')
if details.graphics_subheader_offsets is not None:
for graphics_subhead_num in range(
details.graphics_subheader_offsets.size):
print_func('----- Graphic {} -----'.format(graphics_subhead_num))
hdr = details.parse_graphics_subheader(graphics_subhead_num)
_print_graphics_header(hdr)
data = details.get_graphics_bytes(graphics_subhead_num)
print_func('GSDATA = {}'.format(_decode_effort(data)))
print_func('')
if details.symbol_subheader_offsets is not None:
for symbol_subhead_num in range(details.symbol_subheader_offsets.size):
print_func('----- Symbol {} -----'.format(symbol_subhead_num))
hdr = details.parse_symbol_subheader(symbol_subhead_num)
_print_symbol_header(hdr)
data = details.get_symbol_bytes(symbol_subhead_num)
print_func('SSDATA = {}'.format(_decode_effort(data)))
print_func('')
if details.label_subheader_offsets is not None:
for label_subhead_num in range(details.label_subheader_offsets.size):
print_func('----- Label {} -----'.format(label_subhead_num))
hdr = details.parse_label_subheader(label_subhead_num)
_print_label_header(hdr)
data = details.get_label_bytes(label_subhead_num)
print_func('LSDATA = {}'.format(_decode_effort(data)))
print_func('')
if details.text_subheader_offsets is not None:
for text_subhead_num in range(details.text_subheader_offsets.size):
print_func('----- Text {} -----'.format(text_subhead_num))
hdr = details.parse_text_subheader(text_subhead_num)
_print_text_header(hdr)
data = details.get_text_bytes(text_subhead_num)
print_func('TSDATA = {}'.format(_decode_effort(data)))
print_func('')
if details.des_subheader_offsets is not None:
for des_subhead_num in range(details.des_subheader_offsets.size):
print_func('----- DES {} -----'.format(des_subhead_num))
hdr = details.parse_des_subheader(des_subhead_num)
_print_des_header(hdr)
data = details.get_des_bytes(des_subhead_num)
des_id = hdr.DESID if details.nitf_version == '02.10' else hdr.DESTAG
if des_id.strip() in ['XML_DATA_CONTENT', 'SICD_XML', 'SIDD_XML']:
xml_str = minidom.parseString(data.decode()).toprettyxml(
indent=' ', newl='\n')
# NB: this may or not exhibit platform dependent choices in which codec (i.e. latin-1 versus utf-8)
print_func('DESDATA =')
for line_num, xml_entry in enumerate(xml_str.splitlines()):
if line_num == 0:
# Remove xml that gets inserted by minidom, if it's not actually there
if (not data.startswith(b'<?xml version')
) and xml_entry.startswith('<?xml version'):
continue
print_func(xml_entry)
elif xml_entry.strip() != '':
# Remove extra new lines if XML is already formatted
print_func(xml_entry)
elif des_id.strip() in [
'TRE_OVERFLOW', 'Registered Extensions',
'Controlled Extensions'
]:
tres = TREList.from_bytes(data, 0)
print_func('DESDATA = ')
_print_tres(tres)
else:
# Unknown user-defined data
print_func('DESDATA = {}'.format(_decode_effort(data)))
print_func('')
if details.res_subheader_offsets is not None:
for res_subhead_num in range(details.res_subheader_offsets.size):
print_func('----- RES {} -----'.format(res_subhead_num))
hdr = details.parse_res_subheader(res_subhead_num)
_print_res_header(hdr)
data = details.get_res_bytes(res_subhead_num)
print_func('RESDATA = {}'.format(_decode_effort(data)))
print_func('')
def print_nitf(file_name, dest=sys.stdout):
# Configure print function for desired destination
# - e.g., stdout, string buffer, file
global print_func
print_func = functools.partial(print, file=dest)
details = NITFDetails(file_name)
print_func('----- File Header -----')
print_file_header(details.nitf_header)
print_func('')
if details.img_subheader_offsets is not None:
for i in range(details.img_subheader_offsets.size):
print_func('----- Image {} -----'.format(i))
hdr = details.parse_image_subheader(i)
print_image_header(hdr)
print_func('')
if details.graphics_subheader_offsets is not None:
for i in range(details.graphics_subheader_offsets.size):
print_func('----- Graphic {} -----'.format(i))
hdr = details.parse_graphics_subheader(i)
print_graphics_header(hdr)
data = details.get_graphics_bytes(i)
print_func('GSDATA = {}'.format(_decode_effort(data)))
print_func('')
if details.text_subheader_offsets is not None:
for i in range(details.text_subheader_offsets.size):
print_func('----- Text {} -----'.format(i))
hdr = details.parse_text_subheader(i)
print_text_header(hdr)
data = details.get_text_bytes(i)
print_func('TSDATA = {}'.format(_decode_effort(data)))
print_func('')
if details.des_subheader_offsets is not None:
for i in range(details.des_subheader_offsets.size):
print_func('----- DES {} -----'.format(i))
hdr = details.parse_des_subheader(i)
print_des_header(hdr)
data = details.get_des_bytes(i)
if hdr.DESID.strip() == 'XML_DATA_CONTENT':
xml_str = xml.dom.minidom.parseString(
data.decode()).toprettyxml(indent=' ')
# Remove extra new lines if XML is already formatted
xml_str = os.linesep.join(
[s for s in xml_str.splitlines() if s.strip()])
print_func('DESDATA = {}'.format(xml_str))
elif hdr.DESID.strip() == 'TRE_OVERFLOW':
tres = TREList.from_bytes(data, 0)
print_func('DESDATA = ')
print_tres(tres)
else:
# Unknown user-defined data
print_func('DESDATA = {}'.format(_decode_effort(data)))
print_func('')
if details.res_subheader_offsets is not None:
for i in range(details.res_subheader_offsets.size):
print_func('----- RES {} -----'.format(i))
hdr = details.parse_res_subheader(i)
print_res_header(hdr)
data = details.get_res_bytes(i)
print_func('RESDATA = {}'.format(_decode_effort(data)))
print_func('')