def _create_data_extension_details(self):
super(SICDWriter, self)._create_data_extension_details()
uh_args = self.sicd_meta.get_des_details(self._check_older_version)
desshdt = str(self.sicd_meta.ImageCreation.DateTime.astype('datetime64[s]'))
if desshdt[-1] != 'Z':
desshdt += 'Z'
uh_args['DESSHDT'] = desshdt
desshlpg = ''
if self.sicd_meta.GeoData is not None and self.sicd_meta.GeoData.ImageCorners is not None:
# noinspection PyTypeChecker
icp = self.sicd_meta.GeoData.ImageCorners.get_array(dtype=numpy.float64)
temp = []
for entry in icp:
temp.append('{0:0=+12.8f}{1:0=+13.8f}'.format(entry[0], entry[1]))
temp.append(temp[0])
desshlpg = ''.join(temp)
uh_args['DESSHLPG'] = desshlpg
subhead = DataExtensionHeader(
Security=self._security_tags,
UserHeader=XMLDESSubheader(**uh_args))
self._des_details = (
DESDetails(subhead, self.sicd_meta.to_xml_bytes(tag='SICD', urn=uh_args['DESSHTN'])), )
def _find_sicd(self):
self._is_sicd = False
self._sicd_meta = None
if self.des_subheader_offsets is None:
return
for i in range(self.des_subheader_offsets.size):
subhead_bytes = self.get_des_subheader_bytes(i)
if subhead_bytes.startswith(b'DEXML_DATA_CONTENT'):
des_header = DataExtensionHeader.from_bytes(subhead_bytes, start=0)
des_bytes = self.get_des_bytes(i)
# noinspection PyBroadException
try:
root_node, xml_ns = parse_xml_from_string(des_bytes.decode('utf-8').strip())
if 'SIDD' in root_node.tag: # namespace makes this ugly
# NOTE that SIDD files are supposed to have the corresponding
# SICD xml as one of the DES AFTER the SIDD xml.
# The same basic format is used for both headers.
# So, abandon if we find a SIDD xml
self._des_index = None
self._des_header = None
self._is_sicd = False
break
elif 'SICD' in root_node.tag: # namespace makes this ugly
self._des_index = i
self._des_header = des_header
self._is_sicd = True
self._sicd_meta = SICDType.from_node(root_node, xml_ns, ns_key='default')
break
except Exception:
continue
elif subhead_bytes.startswith(b'DESIDD_XML'):
# This is an old format SIDD and can't be a SICD
self._des_index = None
self._des_header = None
self._is_sicd = False
break
elif subhead_bytes.startswith(b'DESICD_XML'):
# This is an old format SICD
des_bytes = self.get_des_bytes(i)
try:
root_node, xml_ns = parse_xml_from_string(des_bytes)
if 'SICD' in root_node.tag: # namespace makes this ugly
self._des_index = i
self._des_header = None
self._is_sicd = True
self._sicd_meta = SICDType.from_node(root_node, xml_ns, ns_key='default')
break
except Exception as e:
logging.error('We found an apparent old-style SICD DES header, '
'but failed parsing with error {}'.format(e))
continue
if not self._is_sicd:
return
self._sicd_meta.derive()
def find_des():
for i in range(nitf_details.des_subheader_offsets.size):
subhead_bytes = nitf_details.get_des_subheader_bytes(i)
if nitf_details.nitf_version == '02.00':
des_header = DataExtensionHeader0.from_bytes(subhead_bytes,
start=0)
elif nitf_details.nitf_version == '02.10':
des_header = DataExtensionHeader.from_bytes(subhead_bytes,
start=0)
else:
raise ValueError('Got unhandled NITF version {}'.format(
nitf_details.nitf_version))
if subhead_bytes.startswith(b'DEXML_DATA_CONTENT'):
des_bytes = nitf_details.get_des_bytes(i).decode(
'utf-8').strip().encode()
# noinspection PyBroadException
try:
root_node, xml_ns = parse_xml_from_string(des_bytes)
if 'SIDD' in root_node.tag: # namespace makes this ugly
sidd_des.append(
(i, des_bytes, root_node, xml_ns, des_header))
elif 'SICD' in root_node.tag:
sicd_des.append(
(i, des_bytes, root_node, xml_ns, des_header))
except Exception:
continue
elif subhead_bytes.startswith(b'DESIDD_XML'):
# This is an old format SIDD
des_bytes = nitf_details.get_des_bytes(i).decode(
'utf-8').strip().encode()
try:
root_node, xml_ns = parse_xml_from_string(des_bytes)
if 'SIDD' in root_node.tag: # namespace makes this ugly
sidd_des.append(
(i, des_bytes, root_node, xml_ns, des_header))
except Exception as e:
logger.exception(
'SIDD: Old-style SIDD DES header at index {}, but failed parsing'
.format(i))
continue
elif subhead_bytes.startswith(b'DESICD_XML'):
# This is an old format SICD
des_bytes = nitf_details.get_des_bytes(i).decode(
'utf-8').strip().encode()
try:
root_node, xml_ns = parse_xml_from_string(des_bytes)
if 'SICD' in root_node.tag: # namespace makes this ugly
sicd_des.append(
(i, des_bytes, root_node, xml_ns, des_header))
except Exception as e:
logger.exception(
'SIDD: Old-style SICD DES header at index {}, but failed parsing'
.format(i))
continue
def check_data_extension_headers():
# type: () -> (str, Union[DataExtensionHeader, DataExtensionHeader0])
sicd_des = []
for i in range(nitf_details.des_subheader_offsets.size):
subhead_bytes = nitf_details.get_des_subheader_bytes(i)
des_bytes = None
if subhead_bytes.startswith(b'DEXML_DATA_CONTENT'):
des_bytes = nitf_details.get_des_bytes(i)
elif subhead_bytes.startswith(b'DESIDD_XML'):
raise ValueError(
'This file contains an old format SIDD DES, and should be a SIDD file'
)
elif subhead_bytes.startswith(b'DESICD_XML'):
des_bytes = nitf_details.get_des_bytes(i)
if des_bytes is None:
continue
# compare the SICD structure and the des header structure
if nitf_details.nitf_version == '02.00':
des_header = DataExtensionHeader0.from_bytes(subhead_bytes,
start=0)
elif nitf_details.nitf_version == '02.10':
des_header = DataExtensionHeader.from_bytes(subhead_bytes,
start=0)
else:
raise ValueError('Got unhandled NITF version {}'.format(
nitf_details.nitf_version))
try:
des_bytes = des_bytes.decode('utf-8').strip().encode()
root_node, xml_ns = parse_xml_from_string(des_bytes)
# namespace makes this ugly
if 'SIDD' in root_node.tag:
raise ValueError(
'This file contains a SIDD DES, and should be a SIDD file'
)
elif 'SICD' in root_node.tag:
sicd_des.append((i, des_bytes, des_header))
except Exception as e:
logger.error(
'Failed parsing the xml DES entry {} as xml'.format(i))
raise e
if len(sicd_des) == 0:
raise ValueError(
'No SICD DES values found, so this is not a viable SICD file')
elif len(sicd_des) > 1:
raise ValueError('Multiple SICD DES values found at indices {},\n'
'so this is not a viable SICD file'.format(
[entry[0] for entry in sicd_des]))
return sicd_des[0][1], sicd_des[0][2]
def _create_sidd_des_segment(self, index):
"""
Create the details for the sidd data extension at `index`.
Parameters
----------
index : int
Returns
-------
DESDetails
"""
imgs = self._img_groups[index]
security = self.image_details[imgs[0]].subheader.Security
sidd = self.sidd_meta[index]
uh_args = sidd.get_des_details()
try:
desshdt = str(
sidd.ProductCreation.ProcessorInformation.ProcessingDateTime.
astype('datetime64[s]'))
except AttributeError:
desshdt = str(numpy.datetime64('now'))
if desshdt[-1] != 'Z':
desshdt += 'Z'
uh_args['DESSHDT'] = desshdt
desshlpg = ''
icp = self._get_icp(sidd)
if icp is not None:
temp = []
for entry in icp:
temp.append('{0:0=+12.8f}{1:0=+13.8f}'.format(
entry[0], entry[1]))
temp.append(temp[0])
desshlpg = ''.join(temp)
uh_args['DESSHLPG'] = desshlpg
subhead = DataExtensionHeader(Security=security,
UserHeader=XMLDESSubheader(**uh_args))
return DESDetails(subhead, sidd.to_xml_bytes(tag='SIDD'))
def _create_sicd_des_segment(self, index):
"""
Create the details for the sicd data extension at `index`.
Parameters
----------
index : int
Returns
-------
DESDetails
"""
security_tags = self.security_tags
sicd = self.sicd_meta[index]
uh_args = sicd.get_des_details(check_version1_compliance=True)
if sicd.ImageCreation.DateTime is None:
desshdt = datetime.utcnow().isoformat('T', timespec='seconds')
else:
desshdt = str(sicd.ImageCreation.DateTime.astype('datetime64[s]'))
if desshdt[-1] != 'Z':
desshdt += 'Z'
uh_args['DESSHDT'] = desshdt
desshlpg = ''
if sicd.GeoData is not None and sicd.GeoData.ImageCorners is not None:
# noinspection PyTypeChecker
icp = sicd.GeoData.ImageCorners.get_array(dtype=numpy.float64)
temp = []
for entry in icp:
temp.append('{0:0=+12.8f}{1:0=+13.8f}'.format(
entry[0], entry[1]))
temp.append(temp[0])
desshlpg = ''.join(temp)
uh_args['DESSHLPG'] = desshlpg
subhead = DataExtensionHeader(Security=security_tags,
UserHeader=XMLDESSubheader(**uh_args))
return DESDetails(
subhead, sicd.to_xml_bytes(tag='SICD', urn=uh_args['DESSHTN']))
def generic_nitf_header_test(instance, test_file):
assert isinstance(instance, unittest.TestCase)
# can we parse it at all? how long does it take?
with instance.subTest(msg="header parsing"):
start = time.time()
details = NITFDetails(test_file)
# how long does it take?
logging.info('unpacked nitf details in {}'.format(time.time() - start))
# how does it look?
logging.debug(details.nitf_header)
# is the output as long as it should be?
with instance.subTest(msg="header length match"):
header_string = details.nitf_header.to_bytes()
equality = (len(header_string) == details.nitf_header.HL)
if not equality:
logging.error(
'len(produced header) = {}, nitf_header.HL = {}'.format(
len(header_string), details.nitf_header.HL))
instance.assertTrue(equality)
# is the output what it should be?
with instance.subTest(msg="header content match"):
with open(test_file, 'rb') as fi:
file_header = fi.read(details.nitf_header.HL)
equality = (file_header == header_string)
if not equality:
chunk_size = 80
start_chunk = 0
while start_chunk < len(header_string):
end_chunk = min(start_chunk + chunk_size, len(header_string))
logging.error('real[{}:{}] = {}'.format(
start_chunk, end_chunk,
file_header[start_chunk:end_chunk]))
logging.error('prod[{}:{}] = {}'.format(
start_chunk, end_chunk,
header_string[start_chunk:end_chunk]))
start_chunk = end_chunk
instance.assertTrue(equality)
# is each image subheader working?
if details.img_segment_offsets is not None:
for i in range(details.img_segment_offsets.size):
with instance.subTest('image subheader {} match'.format(i)):
img_bytes = details.get_image_subheader_bytes(i)
img_sub = ImageSegmentHeader.from_bytes(img_bytes, start=0)
instance.assertEqual(len(img_bytes),
img_sub.get_bytes_length(),
msg='image subheader as long as expected')
instance.assertEqual(
img_bytes,
img_sub.to_bytes(),
msg=
'image subheader serializes and deserializes as expected')
# is each text segment working?
if details.text_segment_offsets is not None:
for i in range(details.text_segment_offsets.size):
with instance.subTest('text subheader {} match'.format(i)):
txt_bytes = details.get_text_subheader_bytes(i)
txt_sub = TextSegmentHeader.from_bytes(txt_bytes, start=0)
instance.assertEqual(len(txt_bytes),
txt_sub.get_bytes_length(),
msg='text subheader as long as expected')
instance.assertEqual(
txt_bytes,
txt_sub.to_bytes(),
msg='text subheader serializes and deserializes as expected'
)
# is each graphics segment working?
if details.graphics_segment_offsets is not None:
for i in range(details.graphics_segment_offsets.size):
with instance.subTest('graphics subheader {} match'.format(i)):
graphics_bytes = details.get_graphics_subheader_bytes(i)
graphics_sub = GraphicsSegmentHeader.from_bytes(graphics_bytes,
start=0)
instance.assertEqual(
len(graphics_bytes),
graphics_sub.get_bytes_length(),
msg='graphics subheader as long as expected')
instance.assertEqual(
graphics_bytes,
graphics_sub.to_bytes(),
msg=
'graphics subheader serializes and deserializes as expected'
)
# is each data extenson subheader working?
if details.des_segment_offsets is not None:
for i in range(details.des_segment_offsets.size):
with instance.subTest('des subheader {} match'.format(i)):
des_bytes = details.get_des_subheader_bytes(i)
des_sub = DataExtensionHeader.from_bytes(des_bytes, start=0)
instance.assertEqual(len(des_bytes),
des_sub.get_bytes_length(),
msg='des subheader as long as expected')
instance.assertEqual(
des_bytes,
des_sub.to_bytes(),
msg='des subheader serializes and deserializes as expected'
)
