def __init__(self, file_name):
"""
Parameters
----------
file_name : str
"""
if h5py is None:
raise ImportError(
"Can't read ICEYE files, because the h5py dependency is missing."
)
if not os.path.isfile(file_name):
raise SarpyIOError('Path {} is not a file'.format(file_name))
with h5py.File(file_name, 'r') as hf:
if 's_q' not in hf or 's_i' not in hf:
raise SarpyIOError(
'The hdf file does not have the real (s_q) or imaginary dataset (s_i).'
)
if 'satellite_name' not in hf:
raise SarpyIOError(
'The hdf file does not have the satellite_name dataset.')
if 'product_name' not in hf:
raise SarpyIOError(
'The hdf file does not have the product_name dataset.')
self._file_name = file_name
def _get_hdf_dicts(self):
with h5py.File(self._file_name, 'r') as hf:
h5_dict = _extract_attrs(hf)
band_dict = OrderedDict()
shape_dict = OrderedDict()
dtype_dict = OrderedDict()
for gp_name in sorted(hf.keys()):
if self._mission_id == 'CSG' and gp_name == 'LRHM':
continue # this is extraneous
gp = hf[gp_name]
band_dict[gp_name] = OrderedDict()
_extract_attrs(gp, out=band_dict[gp_name])
if self._mission_id in ['CSK', 'KMPS']:
the_dataset = gp['SBI']
elif self._mission_id == 'CSG':
the_dataset = gp['IMG']
else:
raise ValueError('Unhandled mission id {}'.format(
self._mission_id))
_extract_attrs(the_dataset, out=band_dict[gp_name])
shape_dict[gp_name] = the_dataset.shape[:2]
if the_dataset.dtype.name == 'float32':
dtype_dict[gp_name] = 'RE32F_IM32F'
elif the_dataset.dtype.name == 'int16':
dtype_dict[gp_name] = 'RE16I_IM16I'
else:
raise ValueError(
'Got unexpected data type {}, name {}'.format(
the_dataset.dtype, the_dataset.dtype.name))
return h5_dict, band_dict, shape_dict, dtype_dict
def __init__(self, file_name):
"""
Parameters
----------
file_name : str
"""
if h5py is None:
raise ImportError(
"Can't read NISAR files, because the h5py dependency is missing."
)
if not os.path.isfile(file_name):
raise SarpyIOError('Path {} is not a file'.format(file_name))
with h5py.File(file_name, 'r') as hf:
# noinspection PyBroadException
try:
# noinspection PyUnusedLocal
gp = hf['/science/LSAR/SLC']
except:
raise SarpyIOError(
'The hdf5 file does not have required path /science/LSAR/SLC'
)
self._file_name = file_name
def get_sicd_collection(self):
"""
Get the sicd collection for the bands.
Returns
-------
Tuple[Dict[str, SICDType], Dict[str, str], Tuple[bool, bool, bool]]
the first entry is a dictionary of the form {band_name: sicd}
the second entry is of the form {band_name: shape}
the third entry is the symmetry tuple
"""
# TODO: check if the hdf already has the sicds defined, and fish them out if so.
with h5py.File(self.file_name, 'r') as hf:
# fetch the base shared sicd
base_sicd = self._get_base_sicd(hf)
# prepare our output workspace
out_sicds = OrderedDict()
shapes = OrderedDict()
symmetry = (base_sicd.SCPCOA.SideOfTrack == 'L', False, True)
# fetch the common use data for frequency issues
collect_start, collect_end, duration = self._get_collection_times(hf)
zd_time, ss_az_s, grid_r, grid_zd_time = self._get_zero_doppler_data(hf, base_sicd)
gp = hf['/science/LSAR/SLC/metadata/calibrationInformation/geometry']
beta0 = gp['beta0']
gamma0 = gp['gamma0']
sigma0 = gp['sigma0']
# formulate the frequency specific sicd information
freqs = self._get_frequency_list(hf)
for i, freq in enumerate(freqs):
gp_name = '/science/LSAR/SLC/swaths/frequency{}'.format(freq)
gp = hf[gp_name]
freq_sicd, pols, tx_rcv_pol, center_freq = self._get_freq_specific_sicd(gp, base_sicd)
# formulate the frequency dependent doppler grid
# TODO: Future Change Required - processedAzimuthBandwidth acknowledged
# by JPL to be wrong in simulated datasets.
dop_bw = gp['processedAzimuthBandwidth'][()]
gp2 = hf['/science/LSAR/SLC/metadata/processingInformation/parameters/frequency{}'.format(freq)]
dopcentroid_sampled = gp2['dopplerCentroid'][:]
doprate_sampled = gp2['azimuthFMRate'][:]
r_ca_sampled = gp['slantRange'][:]
# formulate the frequency/polarization specific sicd information
for j, pol in enumerate(pols):
ds_name = '{}/{}'.format(gp_name, pol)
ds = gp[pol]
pol_sicd, shape = self._get_pol_specific_sicd(
hf, ds, freq_sicd, pol, freq, j, tx_rcv_pol[j],
r_ca_sampled, zd_time, grid_zd_time, grid_r,
doprate_sampled, dopcentroid_sampled, center_freq,
ss_az_s, dop_bw, beta0, gamma0, sigma0)
out_sicds[ds_name] = pol_sicd
shapes[ds_name] = ds.shape[:2]
return out_sicds, shapes, symmetry
def _read_raw_fun(self, range1, range2):
def validate_gp(gp, name):
if not isinstance(gp, h5py.Dataset):
raise ValueError(
'hdf5 group {} is expected to be a dataset, got type {}'.
format(name, type(gp)))
if len(gp.shape) != 2:
raise ValueError('Dataset {} has unexpected shape {}'.format(
name, gp.shape))
def reorder(tr):
if tr[2] > 0:
return tr, False
else:
if tr[1] == -1 and tr[2] < 0:
return (0, tr[0] + 1, -tr[2]), True
else:
return (tr[1], tr[0], -tr[2]), True
r1, r2 = self._reorder_arguments(range1, range2)
r1, rev1 = reorder(r1)
r2, rev2 = reorder(r2)
with h5py.File(self._file_name, 'r') as hf:
real_gp = hf[self._real_group]
imag_gp = hf[self._imaginary_group]
validate_gp(real_gp, self._real_group)
validate_gp(imag_gp, self._imaginary_group)
real_data = real_gp[r1[0]:r1[1]:r1[2], r2[0]:r2[1]:r2[2]]
data = numpy.zeros((real_data.shape[0], real_data.shape[1], 2),
dtype=real_data.dtype)
data[:, :, 0] = real_data
del real_data
data[:, :, 1] = imag_gp[r1[0]:r1[1]:r1[2], r2[0]:r2[1]:r2[2]]
if rev1 and rev2:
return data[::-1, ::-1]
elif rev1:
return data[::-1, :]
elif rev2:
return data[:, ::-1]
else:
return data
def __init__(self, file_name):
"""
Parameters
----------
file_name : str
"""
if h5py is None:
raise ImportError(
"Can't read Cosmo Skymed files, because the h5py dependency is missing."
)
if not os.path.isfile(file_name):
raise SarpyIOError('Path {} is not a file'.format(file_name))
with h5py.File(file_name, 'r') as hf:
try:
self._mission_id = hf.attrs['Mission ID'].decode('utf-8')
except KeyError:
raise SarpyIOError(
'The hdf file does not have the top level attribute "Mission ID"'
)
try:
self._product_type = hf.attrs['Product Type'].decode('utf-8')
except KeyError:
raise SarpyIOError(
'The hdf file does not have the top level attribute "Product Type"'
)
if self._mission_id not in ['CSK', 'CSG', 'KMPS']:
raise ValueError(
'Expected hdf5 attribute `Mission ID` should be one of "CSK", "CSG", or "KMPS"). '
'Got Mission ID = {}.'.format(self._mission_id))
if 'SCS' not in self._product_type:
raise ValueError(
'Expected hdf to contain complex products '
'(attribute `Product Type` which contains "SCS"). '
'Got Product Type = {}'.format(self._product_type))
self._file_name = file_name
def get_sicd(self):
"""
Gets the SICD structure.
Returns
-------
Tuple[SICDType, tuple, tuple]
The sicd structure, the data size argument, and the symmetry argument.
"""
def get_collection_info():
# type: () -> CollectionInfoType
return CollectionInfoType(
CollectorName=_stringify(hf['satellite_name'][()]),
CoreName=_stringify(hf['product_name'][()]),
CollectType='MONOSTATIC',
Classification='UNCLASSIFIED',
RadarMode=RadarModeType(
ModeType=_stringify(hf['acquisition_mode'][()]).upper(),
ModeID=_stringify(hf['product_type'][()])))
def get_image_creation():
# type: () -> ImageCreationType
from sarpy.__about__ import __version__
return ImageCreationType(
Application='ICEYE_P_{}'.format(hf['processor_version'][()]),
DateTime=_parse_time(hf['processing_time'][()]),
Site='Unknown',
Profile='sarpy {}'.format(__version__))
def get_image_data():
# type: () -> ImageDataType
samp_prec = _stringify(hf['sample_precision'][()])
if samp_prec.upper() == 'INT16':
pixel_type = 'RE16I_IM16I'
elif samp_prec.upper() == 'FLOAT32':
pixel_type = 'RE32F_IM32F'
else:
raise ValueError('Got unhandled sample precision {}'.format(samp_prec))
num_rows = int_func(number_of_range_samples)
num_cols = int_func(number_of_azimuth_samples)
scp_row = int_func(coord_center[0]) - 1
scp_col = int_func(coord_center[1]) - 1
if 0 < scp_col < num_rows-1:
if look_side == 'left':
scp_col = num_cols - scp_col - 1
else:
# early ICEYE processing bug led to nonsensical SCP
scp_col = int_func(num_cols/2.0)
return ImageDataType(
PixelType=pixel_type,
NumRows=num_rows,
NumCols=num_cols,
FirstRow=0,
FirstCol=0,
FullImage=(num_rows, num_cols),
SCPPixel=(scp_row, scp_col))
def get_geo_data():
# type: () -> GeoDataType
# NB: the remainder will be derived.
return GeoDataType(
SCP=SCPType(
LLH=[coord_center[2], coord_center[3], avg_scene_height]))
def get_timeline():
# type: () -> TimelineType
acq_prf = hf['acquisition_prf'][()]
return TimelineType(
CollectStart=start_time,
CollectDuration=duration,
IPP=[IPPSetType(index=0, TStart=0, TEnd=duration,
IPPStart=0, IPPEnd=int_func(round(acq_prf*duration)),
IPPPoly=[0, acq_prf]), ])
def get_position():
# type: () -> PositionType
# fetch the state information
times_str = hf['state_vector_time_utc'][:, 0]
times = numpy.zeros((times_str.shape[0], ), dtype='float64')
positions = numpy.zeros((times.size, 3), dtype='float64')
velocities = numpy.zeros((times.size, 3), dtype='float64')
for i, entry in enumerate(times_str):
times[i] = get_seconds(_parse_time(entry), start_time, precision='us')
positions[:, 0], positions[:, 1], positions[:, 2] = hf['posX'][:], hf['posY'][:], hf['posZ'][:]
velocities[:, 0], velocities[:, 1], velocities[:, 2] = hf['velX'][:], hf['velY'][:], hf['velZ'][:]
# fir the the position polynomial using cross validation
P_x, P_y, P_z = fit_position_xvalidation(times, positions, velocities, max_degree=8)
return PositionType(ARPPoly=XYZPolyType(X=P_x, Y=P_y, Z=P_z))
def get_radar_collection():
# type : () -> RadarCollection
return RadarCollectionType(
TxPolarization=tx_pol,
TxFrequency=(min_freq, max_freq),
Waveform=[WaveformParametersType(TxFreqStart=min_freq,
TxRFBandwidth=tx_bandwidth,
TxPulseLength=hf['chirp_duration'][()],
ADCSampleRate=hf['range_sampling_rate'][()],
RcvDemodType='CHIRP',
RcvFMRate=0,
index=1)],
RcvChannels=[ChanParametersType(TxRcvPolarization=polarization,
index=1)])
def get_image_formation():
# type: () -> ImageFormationType
return ImageFormationType(
TxRcvPolarizationProc=polarization,
ImageFormAlgo='RMA',
TStartProc=0,
TEndProc=duration,
TxFrequencyProc=(min_freq, max_freq),
STBeamComp='NO',
ImageBeamComp='SV',
AzAutofocus='NO',
RgAutofocus='NO',
RcvChanProc=RcvChanProcType(NumChanProc=1, PRFScaleFactor=1, ChanIndices=[1, ]),)
def get_radiometric():
# type: () -> RadiometricType
return RadiometricType(BetaZeroSFPoly=[[float(hf['calibration_factor'][()]), ],])
def calculate_drate_sf_poly():
r_ca_coeffs = numpy.array([r_ca_scp, 1], dtype='float64')
dop_rate_coeffs = hf['doppler_rate_coeffs'][:]
# Prior to ICEYE 1.14 processor, absolute value of Doppler rate was
# provided, not true Doppler rate. Doppler rate should always be negative
if dop_rate_coeffs[0] > 0:
dop_rate_coeffs *= -1
dop_rate_poly = Poly1DType(Coefs=dop_rate_coeffs)
# now adjust to create
t_drate_ca_poly = dop_rate_poly.shift(
t_0=zd_ref_time - rg_time_scp,
alpha=2/speed_of_light, return_poly=False)
return t_drate_ca_poly, -polynomial.polymul(t_drate_ca_poly, r_ca_coeffs)*speed_of_light/(2*center_freq*vm_ca_sq)
def calculate_doppler_polys():
# extract doppler centroid coefficients
dc_estimate_coeffs = hf['dc_estimate_coeffs'][:]
dc_time_str = hf['dc_estimate_time_utc'][:, 0]
dc_zd_times = numpy.zeros((dc_time_str.shape[0], ), dtype='float64')
for i, entry in enumerate(dc_time_str):
dc_zd_times[i] = get_seconds(_parse_time(entry), start_time, precision='us')
# create a sampled doppler centroid
samples = 49 # copied from corresponding matlab, we just need enough for appropriate refitting
# create doppler time samples
diff_time_rg = first_pixel_time - zd_ref_time + \
numpy.linspace(0, number_of_range_samples/range_sampling_rate, samples)
# doppler centroid samples definition
dc_sample_array = numpy.zeros((samples, dc_zd_times.size), dtype='float64')
for i, coeffs in enumerate(dc_estimate_coeffs):
dc_sample_array[:, i] = polynomial.polyval(diff_time_rg, coeffs)
# create arrays for range/azimuth from scp in meters
azimuth_scp_m, range_scp_m = numpy.meshgrid(
col_ss*(dc_zd_times - zd_time_scp)/ss_zd_s,
(diff_time_rg + zd_ref_time - rg_time_scp)*speed_of_light/2)
# fit the doppler centroid sample array
x_order = min(3, range_scp_m.shape[0]-1)
y_order = min(3, range_scp_m.shape[1]-1)
t_dop_centroid_poly, residuals, rank, sing_values = two_dim_poly_fit(
range_scp_m, azimuth_scp_m, dc_sample_array, x_order=x_order, y_order=y_order,
x_scale=1e-3, y_scale=1e-3, rcond=1e-40)
logger.info(
'The dop_centroid_poly fit details:\n\troot mean square '
'residuals = {}\n\trank = {}\n\tsingular values = {}'.format(
residuals, rank, sing_values))
# define and fit the time coa array
doppler_rate_sampled = polynomial.polyval(azimuth_scp_m, drate_ca_poly)
time_coa = dc_zd_times + dc_sample_array/doppler_rate_sampled
t_time_coa_poly, residuals, rank, sing_values = two_dim_poly_fit(
range_scp_m, azimuth_scp_m, time_coa, x_order=x_order, y_order=y_order,
x_scale=1e-3, y_scale=1e-3, rcond=1e-40)
logger.info(
'The time_coa_poly fit details:\n\troot mean square '
'residuals = {}\n\trank = {}\n\tsingular values = {}'.format(
residuals, rank, sing_values))
return t_dop_centroid_poly, t_time_coa_poly
def get_rma():
# type: () -> RMAType
dop_centroid_poly = Poly2DType(Coefs=dop_centroid_poly_coeffs)
dop_centroid_coa = True
if collect_info.RadarMode.ModeType == 'SPOTLIGHT':
dop_centroid_poly = None
dop_centroid_coa = None
# NB: DRateSFPoly is defined as a function of only range - reshape appropriately
inca = INCAType(
R_CA_SCP=r_ca_scp,
FreqZero=center_freq,
DRateSFPoly=Poly2DType(Coefs=numpy.reshape(drate_sf_poly_coefs, (-1, 1))),
DopCentroidPoly=dop_centroid_poly,
DopCentroidCOA=dop_centroid_coa,
TimeCAPoly=Poly1DType(Coefs=time_ca_poly_coeffs))
return RMAType(
RMAlgoType='OMEGA_K',
INCA=inca)
def get_grid():
# type: () -> GridType
time_coa_poly = Poly2DType(Coefs=time_coa_poly_coeffs)
if collect_info.RadarMode.ModeType == 'SPOTLIGHT':
time_coa_poly = Poly2DType(Coefs=[[float(time_coa_poly_coeffs[0, 0]), ], ])
row_win = _stringify(hf['window_function_range'][()])
if row_win == 'NONE':
row_win = 'UNIFORM'
row = DirParamType(
SS=row_ss,
Sgn=-1,
KCtr=2*center_freq/speed_of_light,
ImpRespBW=2*tx_bandwidth/speed_of_light,
DeltaKCOAPoly=Poly2DType(Coefs=[[0,]]),
WgtType=WgtTypeType(WindowName=row_win))
col_win = _stringify(hf['window_function_azimuth'][()])
if col_win == 'NONE':
col_win = 'UNIFORM'
col = DirParamType(
SS=col_ss,
Sgn=-1,
KCtr=0,
ImpRespBW=col_imp_res_bw,
WgtType=WgtTypeType(WindowName=col_win),
DeltaKCOAPoly=Poly2DType(Coefs=dop_centroid_poly_coeffs*ss_zd_s/col_ss))
return GridType(
Type='RGZERO',
ImagePlane='SLANT',
TimeCOAPoly=time_coa_poly,
Row=row,
Col=col)
def correct_scp():
scp_pixel = sicd.ImageData.SCPPixel.get_array()
scp_ecf = sicd.project_image_to_ground(scp_pixel, projection_type='HAE')
sicd.update_scp(scp_ecf, coord_system='ECF')
with h5py.File(self._file_name, 'r') as hf:
# some common use variables
look_side = _stringify(hf['look_side'][()])
coord_center = hf['coord_center'][:]
avg_scene_height = float(hf['avg_scene_height'][()])
start_time = _parse_time(hf['acquisition_start_utc'][()])
end_time = _parse_time(hf['acquisition_end_utc'][()])
duration = get_seconds(end_time, start_time, precision='us')
center_freq = float(hf['carrier_frequency'][()])
tx_bandwidth = float(hf['chirp_bandwidth'][()])
min_freq = center_freq-0.5*tx_bandwidth
max_freq = center_freq+0.5*tx_bandwidth
pol_temp = _stringify(hf['polarization'][()])
tx_pol = pol_temp[0]
rcv_pol = pol_temp[1]
polarization = tx_pol + ':' + rcv_pol
first_pixel_time = float(hf['first_pixel_time'][()])
near_range = first_pixel_time*speed_of_light/2
number_of_range_samples = float(hf['number_of_range_samples'][()])
number_of_azimuth_samples = float(hf['number_of_azimuth_samples'][()])
range_sampling_rate = float(hf['range_sampling_rate'][()])
row_ss = speed_of_light/(2*range_sampling_rate)
# define the sicd elements
collect_info = get_collection_info()
image_creation = get_image_creation()
image_data = get_image_data()
geo_data = get_geo_data()
timeline = get_timeline()
position = get_position()
radar_collection = get_radar_collection()
image_formation = get_image_formation()
radiometric = get_radiometric()
# calculate some zero doppler parameters
ss_zd_s = float(hf['azimuth_time_interval'][()])
if look_side == 'left':
ss_zd_s *= -1
zero_doppler_left = _parse_time(hf['zerodoppler_end_utc'][()])
else:
zero_doppler_left = _parse_time(hf['zerodoppler_start_utc'][()])
dop_bw = hf['total_processed_bandwidth_azimuth'][()]
zd_time_scp = get_seconds(zero_doppler_left, start_time, precision='us') + \
image_data.SCPPixel.Col*ss_zd_s
zd_ref_time = first_pixel_time + number_of_range_samples/(2*range_sampling_rate)
vel_scp = position.ARPPoly.derivative_eval(zd_time_scp, der_order=1)
vm_ca_sq = numpy.sum(vel_scp*vel_scp)
rg_time_scp = first_pixel_time + image_data.SCPPixel.Row/range_sampling_rate
r_ca_scp = rg_time_scp*speed_of_light/2
# calculate the doppler rate sf polynomial
drate_ca_poly, drate_sf_poly_coefs = calculate_drate_sf_poly()
# calculate some doppler dependent grid parameters
col_ss = float(numpy.sqrt(vm_ca_sq)*abs(ss_zd_s)*drate_sf_poly_coefs[0])
col_imp_res_bw = dop_bw*abs(ss_zd_s)/col_ss
time_ca_poly_coeffs = [zd_time_scp, ss_zd_s/col_ss]
# calculate the doppler polynomials
dop_centroid_poly_coeffs, time_coa_poly_coeffs = calculate_doppler_polys()
# finish definition of sicd elements
rma = get_rma()
grid = get_grid()
sicd = SICDType(
CollectionInfo=collect_info,
ImageCreation=image_creation,
ImageData=image_data,
GeoData=geo_data,
Timeline=timeline,
Position=position,
RadarCollection=radar_collection,
ImageFormation=image_formation,
Radiometric=radiometric,
RMA=rma,
Grid=grid)
# adjust the scp location
correct_scp()
# derive sicd fields
sicd.derive()
# TODO: RNIIRS?
data_size = (image_data.NumCols, image_data.NumRows)
symmetry = (True, False, True) if look_side == 'left' else (False, False, True)
return sicd, data_size, symmetry