def update_object(outFile, inObj, box, updateMode=True):
"""Do not write h5 file if: 1) h5 exists and readable,
2) it contains all date12 from ifgramStackDict,
or all datasets from geometryDict"""
write_flag = True
if updateMode and ut.run_or_skip(outFile, check_readable=True) == 'skip':
if inObj.name == 'ifgramStack':
in_size = inObj.get_size(box=box)[1:]
in_date12_list = inObj.get_date12_list()
outObj = ifgramStack(outFile)
out_size = outObj.get_size()[1:]
out_date12_list = outObj.get_date12_list(dropIfgram=False)
if out_size == in_size and set(in_date12_list).issubset(set(out_date12_list)):
print(('All date12 exists in file {} with same size as required,'
' no need to re-load.'.format(os.path.basename(outFile))))
write_flag = False
elif inObj.name == 'geometry':
outObj = geometry(outFile)
outObj.open(print_msg=False)
if (outObj.get_size() == inObj.get_size(box=box)
and all(i in outObj.datasetNames for i in inObj.get_dataset_list())):
print(('All datasets exists in file {} with same size as required,'
' no need to re-load.'.format(os.path.basename(outFile))))
write_flag = False
return write_flag
def read_geometry(inps):
ts_obj = timeseries(inps.timeseries_file)
ts_obj.open(print_msg=False)
# 2D / 3D geometry
if inps.geom_file:
geom_obj = geometry(inps.geom_file)
geom_obj.open()
if 'incidenceAngle' not in geom_obj.datasetNames:
inps.incAngle = ut.incidence_angle(ts_obj.metadata, dimension=0)
inps.rangeDist = ut.range_distance(ts_obj.metadata, dimension=0)
else:
print(('read 2D incidenceAngle,slantRangeDistance from {} file:'
' {}').format(geom_obj.name, os.path.basename(geom_obj.file)))
inps.incAngle = geom_obj.read(datasetName='incidenceAngle', print_msg=False).flatten()
inps.rangeDist = geom_obj.read(datasetName='slantRangeDistance', print_msg=False).flatten()
if 'bperp' in geom_obj.datasetNames:
print('read 3D bperp from {} file: {} ...'.format(geom_obj.name, os.path.basename(geom_obj.file)))
dset_list = ['bperp-{}'.format(d) for d in ts_obj.dateList]
inps.pbase = geom_obj.read(datasetName=dset_list, print_msg=False).reshape((ts_obj.numDate, -1))
inps.pbase -= np.tile(inps.pbase[ts_obj.refIndex, :].reshape(1, -1), (ts_obj.numDate, 1))
else:
print('read mean bperp from {} file'.format(ts_obj.name))
inps.pbase = ts_obj.pbase.reshape((-1, 1))
# 0D geometry
else:
print('read mean incidenceAngle,slantRangeDistance,bperp value from {} file'.format(ts_obj.name))
inps.incAngle = ut.incidence_angle(ts_obj.metadata, dimension=0)
inps.rangeDist = ut.range_distance(ts_obj.metadata, dimension=0)
inps.pbase = ts_obj.pbase.reshape((-1, 1))
inps.sinIncAngle = np.sin(inps.incAngle * np.pi / 180.)
return inps
def update_object(outFile, inObj, box, updateMode=True):
"""Do not write h5 file if: 1) h5 exists and readable,
2) it contains all date12 from ifgramStackDict,
or all datasets from geometryDict"""
write_flag = True
if updateMode and ut.run_or_skip(outFile, check_readable=True) == 'skip':
if inObj.name == 'ifgramStack':
in_size = inObj.get_size(box=box)[1:]
in_date12_list = inObj.get_date12_list()
outObj = ifgramStack(outFile)
out_size = outObj.get_size()[1:]
out_date12_list = outObj.get_date12_list(dropIfgram=False)
if out_size == in_size and set(in_date12_list).issubset(set(out_date12_list)):
print(('All date12 exists in file {} with same size as required,'
' no need to re-load.'.format(os.path.basename(outFile))))
write_flag = False
elif inObj.name == 'geometry':
outObj = geometry(outFile)
outObj.open(print_msg=False)
if (outObj.get_size() == inObj.get_size(box=box)
and all(i in outObj.datasetNames for i in inObj.get_dataset_list())):
print(('All datasets exists in file {} with same size as required,'
' no need to re-load.'.format(os.path.basename(outFile))))
write_flag = False
return write_flag
def run_or_skip(inps, dsNameDict, out_file):
flag = 'run'
# check 1 - update mode status
if not inps.updateMode:
return flag
# check 2 - output file existance
if ut.run_or_skip(out_file, check_readable=True) == 'run':
return flag
# check 3 - output dataset info
key = [i for i in ['unwrapPhase', 'height'] if i in dsNameDict.keys()][0]
ds_shape = dsNameDict[key][1]
in_shape = ds_shape[-2:]
if 'unwrapPhase' in dsNameDict.keys():
# compare date12 and size
ds = gdal.Open(inps.unwFile, gdal.GA_ReadOnly)
in_date12_list = [
ds.GetRasterBand(i + 1).GetMetadata("unwrappedPhase")['Dates']
for i in range(ds_shape[0])
]
in_date12_list = ['_'.join(d.split('_')[::-1]) for d in in_date12_list]
try:
out_obj = ifgramStack(out_file)
out_obj.open(print_msg=False)
out_shape = (out_obj.length, out_obj.width)
out_date12_list = out_obj.get_date12_list(dropIfgram=False)
if out_shape == in_shape and set(in_date12_list).issubset(
set(out_date12_list)):
print((
'All date12 exists in file {} with same size as required,'
' no need to re-load.'.format(os.path.basename(out_file))))
flag = 'skip'
except:
pass
elif 'height' in dsNameDict.keys():
# compare dataset names and size
in_dsNames = list(dsNameDict.keys())
in_size = in_shape[0] * in_shape[1] * 4 * len(in_dsNames)
out_obj = geometry(out_file)
out_obj.open(print_msg=False)
out_dsNames = out_obj.datasetNames
out_shape = (out_obj.length, out_obj.width)
out_size = os.path.getsize(out_file)
if (set(in_dsNames).issubset(set(out_dsNames))
and out_shape == in_shape and out_size > in_size * 0.3):
print(('All datasets exists in file {} with same size as required,'
' no need to re-load.'.format(os.path.basename(out_file))))
flag = 'skip'
return flag
def read_geometry(ts_file, geom_file=None, box=None):
"""Read the following geometry info in 0/2/3D
Parameters: ts_file - str, path of time-series file
geom_file - str, path of geometry file
box - tuple of 4 int for (x0, y0, x1, y1) of the area of interest
Returns: sin_inc_angle - 0/2D array, sin(inc_angle)
range_dist - 0/2D array, slant range distance in meter
pbase - 0/3D array, perp baseline in meter
"""
ts_obj = timeseries(ts_file)
ts_obj.open(print_msg=False)
# 0/2/3D geometry
if geom_file:
geom_obj = geometry(geom_file)
geom_obj.open()
# 0/2D incidence angle / slant range distance
if 'incidenceAngle' not in geom_obj.datasetNames:
inc_angle = ut.incidence_angle(ts_obj.metadata, dimension=0)
range_dist = ut.range_distance(ts_obj.metadata, dimension=0)
else:
print(
'read 2D incidenceAngle, slantRangeDistance from {} file: {}'.
format(geom_obj.name, os.path.basename(geom_obj.file)))
inc_angle = geom_obj.read(datasetName='incidenceAngle',
box=box,
print_msg=False).flatten()
range_dist = geom_obj.read(datasetName='slantRangeDistance',
box=box,
print_msg=False).flatten()
# 0/3D perp baseline
if 'bperp' in geom_obj.datasetNames:
print('read 3D bperp from {} file: {} ...'.format(
geom_obj.name, os.path.basename(geom_obj.file)))
dset_list = ['bperp-{}'.format(d) for d in ts_obj.dateList]
pbase = geom_obj.read(datasetName=dset_list,
box=box,
print_msg=False).reshape(
(ts_obj.numDate, -1))
pbase -= np.tile(pbase[ts_obj.refIndex, :].reshape(1, -1),
(ts_obj.numDate, 1))
else:
print('read mean bperp from {} file'.format(ts_obj.name))
pbase = ts_obj.pbase.reshape((-1, 1))
# 0D geometry
else:
print(
'read mean incidenceAngle, slantRangeDistance, bperp value from {} file'
.format(ts_obj.name))
inc_angle = ut.incidence_angle(ts_obj.metadata, dimension=0)
range_dist = ut.range_distance(ts_obj.metadata, dimension=0)
pbase = ts_obj.pbase.reshape((-1, 1))
sin_inc_angle = np.sin(inc_angle * np.pi / 180.)
return sin_inc_angle, range_dist, pbase
def split2boxes(ts_file,
geom_file=None,
memory_size=4,
num_step=0,
print_msg=True):
"""Split into chunks in rows to reduce memory usage
Parameters: ts_file - str, path of time-series h5 file
memory_size - float, max memory to use in GB
print_msg - bool
Returns: box_list - list of tuple of 4 int
num_box - int, number of boxes
"""
ts_obj = timeseries(ts_file)
ts_obj.open(print_msg=False)
length, width = ts_obj.length, ts_obj.width
# 1st dimension size: ts (obs / cor / res / step) + dem_err/inc_angle/rg_dist (+pbase)
num_epoch = ts_obj.numDate * 3 + num_step + 3
if geom_file:
geom_obj = geometry(geom_file)
geom_obj.open(print_msg=False)
if 'bperp' in geom_obj.datasetNames:
num_epoch += ts_obj.numDate
# split in lines based on the input memory limit
y_step = (memory_size * (1e3**3)) / (num_epoch * width * 4)
# calibrate based on experience
y_step = int(ut.round_to_1(y_step * 0.7))
num_box = int((length - 1) / y_step) + 1
if print_msg and num_box > 1:
print('maximum memory size: %.1E GB' % memory_size)
print('split %d lines into %d patches for processing' %
(length, num_box))
print(' with each patch up to %d lines' % y_step)
# y_step / num_box --> box_list
box_list = []
for i in range(num_box):
y0 = i * y_step
y1 = min([length, y0 + y_step])
box = (0, y0, width, y1)
box_list.append(box)
return box_list, num_box
def get_delay_timeseries(inps, atr):
"""Calculate delay time-series and write it to HDF5 file.
Parameters: inps : namespace, all input parameters
atr : dict, metadata to be saved in trop_file
Returns: trop_file : str, file name of ECMWF.h5
"""
def get_dataset_size(fname):
atr = readfile.read_attribute(fname)
return (atr['LENGTH'], atr['WIDTH'])
# check 1 - existing tropo delay file
if (ut.run_or_skip(out_file=inps.trop_file, in_file=inps.grib_file_list, print_msg=False) == 'skip'
and get_dataset_size(inps.trop_file) == get_dataset_size(inps.geom_file)):
print('{} file exists and is newer than all GRIB files, skip updating.'.format(inps.trop_file))
return
# check 2 - geometry file
if any(i is None for i in [inps.geom_file, inps.ref_yx]):
print('No DEM / incidenceAngle / ref_yx found, skip calculating tropospheric delays.')
if not os.path.isfile(inps.trop_file):
inps.trop_file = None
return
# prepare geometry data
geom_obj = geometry(inps.geom_file)
geom_obj.open()
inps.dem = geom_obj.read(datasetName='height')
inps.inc = geom_obj.read(datasetName='incidenceAngle')
if 'latitude' in geom_obj.datasetNames:
inps.lat = geom_obj.read(datasetName='latitude')
inps.lon = geom_obj.read(datasetName='longitude')
else:
inps.lat, inps.lon = get_lat_lon(geom_obj.metadata)
# calculate phase delay
length, width = int(atr['LENGTH']), int(atr['WIDTH'])
num_date = len(inps.grib_file_list)
date_list = [str(re.findall('\d{8}', i)[0]) for i in inps.grib_file_list]
trop_data = np.zeros((num_date, length, width), np.float32)
#trop_data_abs = np.zeros((num_date, length, width), np.float32)
print('calcualting delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...')
print('number of grib files used: {}'.format(num_date))
if verbose:
prog_bar = ptime.progressBar(maxValue=num_date)
for i in range(num_date):
grib_file = inps.grib_file_list[i]
trop_data[i] = get_delay(grib_file, inps)
#trop_data_abs[i] = get_delay_abs(grib_file, inps)
if verbose:
prog_bar.update(i+1, suffix=os.path.basename(grib_file))
if verbose:
prog_bar.close()
# Convert relative phase delay on reference date
inps.ref_date = atr.get('REF_DATE', date_list[0])
print('convert to relative phase delay with reference date: '+inps.ref_date)
inps.ref_idx = date_list.index(inps.ref_date)
trop_data -= np.tile(trop_data[inps.ref_idx, :, :], (num_date, 1, 1))
# Write tropospheric delay to HDF5
atr['REF_Y'] = inps.ref_yx[0]
atr['REF_X'] = inps.ref_yx[1]
ts_obj = timeseries(inps.trop_file)
ts_obj.write2hdf5(data=trop_data,
dates=date_list,
metadata=atr,
refFile=inps.timeseries_file)
#ts_obj.write2hdf5(data=trop_data,
# outFile = 'ERA5_abs.h5',
# dates=date_list,
# metadata=atr,
# refFile=inps.timeseries_file)
return
def get_delay_timeseries(inps, atr):
"""Calculate delay time-series and write it to HDF5 file.
Parameters: inps : namespace, all input parameters
atr : dict, metadata to be saved in trop_file
Returns: trop_file : str, file name of ECMWF.h5
"""
def get_dataset_size(fname):
atr = readfile.read_attribute(fname)
return (atr['LENGTH'], atr['WIDTH'])
# check 1 - existing tropo delay file
if (ut.run_or_skip(out_file=inps.trop_file, in_file=inps.grib_file_list, print_msg=False) == 'skip'
and get_dataset_size(inps.trop_file) == get_dataset_size(inps.geom_file)):
print('{} file exists and is newer than all GRIB files, skip updating.'.format(inps.trop_file))
return
# check 2 - geometry file
if any(i is None for i in [inps.geom_file, inps.ref_yx]):
print('No DEM / incidenceAngle / ref_yx found, skip calculating tropospheric delays.')
if not os.path.isfile(inps.trop_file):
inps.trop_file = None
return
# prepare geometry data
geom_obj = geometry(inps.geom_file)
geom_obj.open()
inps.dem = geom_obj.read(datasetName='height')
inps.inc = geom_obj.read(datasetName='incidenceAngle')
if 'latitude' in geom_obj.datasetNames:
inps.lat = geom_obj.read(datasetName='latitude')
inps.lon = geom_obj.read(datasetName='longitude')
else:
inps.lat, inps.lon = get_lat_lon(geom_obj.metadata)
# calculate phase delay
length, width = int(atr['LENGTH']), int(atr['WIDTH'])
num_date = len(inps.grib_file_list)
date_list = [str(re.findall('\d{8}', i)[0]) for i in inps.grib_file_list]
trop_data = np.zeros((num_date, length, width), np.float32)
print('calcualting delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...')
print('number of grib files used: {}'.format(num_date))
prog_bar = ptime.progressBar(maxValue=num_date)
for i in range(num_date):
grib_file = inps.grib_file_list[i]
trop_data[i] = get_delay(grib_file, inps)
prog_bar.update(i+1, suffix=os.path.basename(grib_file))
prog_bar.close()
# Convert relative phase delay on reference date
inps.ref_date = atr.get('REF_DATE', date_list[0])
print('convert to relative phase delay with reference date: '+inps.ref_date)
inps.ref_idx = date_list.index(inps.ref_date)
trop_data -= np.tile(trop_data[inps.ref_idx, :, :], (num_date, 1, 1))
# Write tropospheric delay to HDF5
atr['REF_Y'] = inps.ref_yx[0]
atr['REF_X'] = inps.ref_yx[1]
ts_obj = timeseries(inps.trop_file)
ts_obj.write2hdf5(data=trop_data,
dates=date_list,
metadata=atr,
refFile=inps.timeseries_file)
return
def get_slice_list(fname):
"""Get list of 2D slice existed in file (for display)"""
fbase, fext = os.path.splitext(os.path.basename(fname))
fext = fext.lower()
atr = read_attribute(fname)
k = atr['FILE_TYPE']
global slice_list
# HDF5 Files
if fext in ['.h5', '.he5']:
with h5py.File(fname, 'r') as f:
d1_list = [i for i in f.keys() if isinstance(f[i], h5py.Dataset)]
if k == 'timeseries' and k in d1_list:
obj = timeseries(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['geometry'] and k not in d1_list:
obj = geometry(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['ifgramStack']:
obj = ifgramStack(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['HDFEOS']:
obj = HDFEOS(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['giantTimeseries']:
obj = giantTimeseries(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['giantIfgramStack']:
obj = giantIfgramStack(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
else:
## Find slice by walking through the file structure
length, width = int(atr['LENGTH']), int(atr['WIDTH'])
def get_hdf5_2d_dataset(name, obj):
global slice_list
if isinstance(obj, h5py.Dataset) and obj.shape[-2:] == (length,
width):
if obj.ndim == 2:
slice_list.append(name)
else:
warnings.warn(
'file has un-defined {}D dataset: {}'.format(
obj.ndim, name))
slice_list = []
with h5py.File(fname, 'r') as f:
f.visititems(get_hdf5_2d_dataset)
# Binary Files
else:
if fext.lower() in ['.trans', '.utm_to_rdc']:
slice_list = ['rangeCoord', 'azimuthCoord']
elif fbase.startswith('los'):
slice_list = ['incidenceAngle', 'azimuthAngle']
elif atr.get('number_bands', '1') == '2' and 'unw' not in k:
slice_list = ['band1', 'band2']
else:
slice_list = ['']
return slice_list
def get_slice_list(fname):
"""Get list of 2D slice existed in file (for display)"""
fbase, fext = os.path.splitext(os.path.basename(fname))
fext = fext.lower()
atr = read_attribute(fname)
k = atr['FILE_TYPE']
global slice_list
# HDF5 Files
if fext in ['.h5', '.he5']:
with h5py.File(fname, 'r') as f:
d1_list = [i for i in f.keys() if isinstance(f[i], h5py.Dataset)]
if k == 'timeseries' and k in d1_list:
obj = timeseries(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['geometry'] and k not in d1_list:
obj = geometry(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['ifgramStack']:
obj = ifgramStack(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['HDFEOS']:
obj = HDFEOS(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['giantTimeseries']:
obj = giantTimeseries(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['giantIfgramStack']:
obj = giantIfgramStack(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
else:
## Find slice by walking through the file structure
length, width = int(atr['LENGTH']), int(atr['WIDTH'])
def get_hdf5_2d_dataset(name, obj):
global slice_list
if isinstance(obj, h5py.Dataset) and obj.shape[-2:] == (length, width):
if obj.ndim == 2:
slice_list.append(name)
else:
warnings.warn('file has un-defined {}D dataset: {}'.format(obj.ndim, name))
slice_list = []
with h5py.File(fname, 'r') as f:
f.visititems(get_hdf5_2d_dataset)
# Binary Files
else:
if fext.lower() in ['.trans', '.utm_to_rdc']:
slice_list = ['rangeCoord', 'azimuthCoord']
elif fbase.startswith('los'):
slice_list = ['incidenceAngle', 'azimuthAngle']
elif atr.get('number_bands', '1') == '2' and 'unw' not in k:
slice_list = ['band1', 'band2']
else:
slice_list = ['']
return slice_list
def check_inputs(inps):
parser = create_parser()
# output directories/files
atr = dict()
mintpy_dir = None
if inps.timeseries_file:
atr = readfile.read_attribute(inps.timeseries_file)
mintpy_dir = os.path.dirname(inps.timeseries_file)
if not inps.outfile:
fbase = os.path.splitext(inps.timeseries_file)[0]
inps.outfile = '{}_{}.h5'.format(fbase, inps.trop_model)
elif inps.geom_file:
atr = readfile.read_attribute(inps.geom_file)
mintpy_dir = os.path.join(os.path.dirname(inps.geom_file), '..')
else:
mintpy_dir = os.path.abspath(os.getcwd())
# trop_file
inps.trop_file = os.path.join(mintpy_dir, 'inputs/{}.h5'.format(inps.trop_model))
print('output tropospheric delay file: {}'.format(inps.trop_file))
# hour
if not inps.hour:
if 'CENTER_LINE_UTC' in atr.keys():
inps.hour = ptime.closest_weather_product_time(atr['CENTER_LINE_UTC'], inps.trop_model)
else:
parser.print_usage()
raise Exception('no input for hour')
print('time of cloest available product: {}:00 UTC'.format(inps.hour))
# date list
if inps.timeseries_file:
print('read date list from timeseries file: {}'.format(inps.timeseries_file))
ts_obj = timeseries(inps.timeseries_file)
ts_obj.open(print_msg=False)
inps.date_list = ts_obj.dateList
elif len(inps.date_list) == 1:
if os.path.isfile(inps.date_list[0]):
print('read date list from text file: {}'.format(inps.date_list[0]))
inps.date_list = ptime.yyyymmdd(np.loadtxt(inps.date_list[0],
dtype=bytes,
usecols=(0,)).astype(str).tolist())
else:
parser.print_usage()
raise Exception('ERROR: input date list < 2')
# Grib data directory
inps.grib_dir = os.path.join(inps.weather_dir, inps.trop_model)
if not os.path.isdir(inps.grib_dir):
os.makedirs(inps.grib_dir)
print('making directory: '+inps.grib_dir)
# Date list to grib file list
inps.grib_file_list = date_list2grib_file(inps.date_list,
inps.hour,
inps.trop_model,
inps.grib_dir)
if 'REF_Y' in atr.keys():
inps.ref_yx = [int(atr['REF_Y']), int(atr['REF_X'])]
print('reference pixel: {}'.format(inps.ref_yx))
# Coordinate system: geocoded or not
inps.geocoded = False
if 'Y_FIRST' in atr.keys():
inps.geocoded = True
print('geocoded: {}'.format(inps.geocoded))
# Prepare DEM, inc_angle, lat/lon file for PyAPS to read
if inps.geom_file:
geom_obj = geometry(inps.geom_file)
geom_obj.open()
print('converting DEM/incAngle for PyAPS to read')
# DEM
dem = readfile.read(inps.geom_file, datasetName='height', print_msg=False)[0]
inps.dem_file = 'pyapsDem.hgt'
writefile.write(dem, inps.dem_file, metadata=atr)
# inc_angle
if 'incidenceAngle' in geom_obj.datasetNames:
inps.inc_angle = readfile.read(inps.geom_file, datasetName='incidenceAngle', print_msg=False)[0]
else:
atr = readfile.read_attribute(inps.timeseries_file)
inps.inc_angle = ut.incidence_angle(atr, dem=dem, dimension=0)
inps.inc_angle = np.ones(dem.shape, dtype=np.float32) * inps.inc_angle
inps.inc_angle_file = 'pyapsIncAngle.flt'
writefile.write(inps.inc_angle, inps.inc_angle_file, metadata=atr)
# latitude
if 'latitude' in geom_obj.datasetNames:
data = readfile.read(inps.geom_file, datasetName='latitude', print_msg=False)[0]
print('converting lat for PyAPS to read')
inps.lat_file = 'pyapsLat.flt'
writefile.write(data, inps.lat_file, metadata=atr)
else:
inps.lat_file = None
# longitude
if 'longitude' in geom_obj.datasetNames:
data = readfile.read(inps.geom_file, datasetName='longitude', print_msg=False)[0]
print('converting lon for PyAPS to read')
inps.lon_file = 'pyapsLon.flt'
writefile.write(data, inps.lon_file, metadata=atr)
else:
inps.lon_file = None
return inps, atr
def correct_dem_error(inps):
"""Correct DEM error of input timeseries file"""
start_time = time.time()
# limit the number of threads to 1
# for slight speedup and big CPU usage save
num_threads_dict = cluster.set_num_threads("1")
## 1. input info
# 1.1 read date info
ts_obj = timeseries(inps.timeseries_file)
ts_obj.open()
num_date = ts_obj.numDate
length, width = ts_obj.length, ts_obj.width
num_step = len(inps.stepFuncDate)
# exclude dates
date_flag = read_exclude_date(inps.excludeDate, ts_obj.dateList)[0]
if inps.polyOrder > np.sum(date_flag):
raise ValueError(
"input poly order {} > number of acquisition {}! Reduce it!".
format(inps.polyOrder, np.sum(date_flag)))
# 1.2 design matrix part 1 - time func for surface deformation
G_defo = get_design_matrix4defo(inps)
## 2. prepare output
# 2.1 metadata
meta = dict(ts_obj.metadata)
print(
'add/update the following configuration metadata to file:\n{}'.format(
configKeys))
for key in configKeys:
meta[key_prefix + key] = str(vars(inps)[key])
# 2.2 instantiate est. DEM error
dem_err_file = 'demErr.h5'
meta['FILE_TYPE'] = 'dem'
meta['UNIT'] = 'm'
ds_name_dict = {'dem': [np.float32, (length, width), None]}
writefile.layout_hdf5(dem_err_file, ds_name_dict, metadata=meta)
# 2.3 instantiate corrected time-series
ts_cor_file = inps.outfile
meta['FILE_TYPE'] = 'timeseries'
writefile.layout_hdf5(ts_cor_file,
metadata=meta,
ref_file=inps.timeseries_file)
# 2.4 instantiate residual phase time-series
ts_res_file = os.path.join(os.path.dirname(inps.outfile),
'timeseriesResidual.h5')
writefile.layout_hdf5(ts_res_file,
metadata=meta,
ref_file=inps.timeseries_file)
## 3. run the estimation and write to disk
# 3.1 split ts_file into blocks to save memory
# 1st dimension size: ts (obs / cor / res / step) + dem_err/inc_angle/rg_dist (+pbase)
num_epoch = num_date * 3 + num_step + 3
if inps.geom_file:
geom_obj = geometry(inps.geom_file)
geom_obj.open(print_msg=False)
if 'bperp' in geom_obj.datasetNames:
num_epoch += num_date
# split in row/line direction based on the input memory limit
num_box = int(
np.ceil((num_epoch * length * width * 4) * 2.5 /
(inps.maxMemory * 1024**3)))
box_list = cluster.split_box2sub_boxes(box=(0, 0, width, length),
num_split=num_box,
dimension='y')
# 3.2 prepare the input arguments for *_patch()
data_kwargs = {
'G_defo': G_defo,
'ts_file': inps.timeseries_file,
'geom_file': inps.geom_file,
'date_flag': date_flag,
'phase_velocity': inps.phaseVelocity,
}
# 3.3 invert / write block-by-block
for i, box in enumerate(box_list):
box_wid = box[2] - box[0]
box_len = box[3] - box[1]
if num_box > 1:
print('\n------- processing patch {} out of {} --------------'.
format(i + 1, num_box))
print('box width: {}'.format(box_wid))
print('box length: {}'.format(box_len))
# update box argument in the input data
data_kwargs['box'] = box
# invert
if not inps.cluster:
# non-parallel
delta_z, ts_cor, ts_res = correct_dem_error_patch(
**data_kwargs)[:-1]
else:
# parallel
print('\n\n------- start parallel processing using Dask -------')
# initiate the output data
delta_z = np.zeros((box_len, box_wid), dtype=np.float32)
ts_cor = np.zeros((num_date, box_len, box_wid), dtype=np.float32)
ts_res = np.zeros((num_date, box_len, box_wid), dtype=np.float32)
# initiate dask cluster and client
cluster_obj = cluster.DaskCluster(inps.cluster,
inps.numWorker,
config_name=inps.config)
cluster_obj.open()
# run dask
delta_z, ts_cor, ts_res = cluster_obj.run(
func=correct_dem_error_patch,
func_data=data_kwargs,
results=[delta_z, ts_cor, ts_res])
# close dask cluster and client
cluster_obj.close()
print('------- finished parallel processing -------\n\n')
# write the block to disk
# with 3D block in [z0, z1, y0, y1, x0, x1]
# and 2D block in [y0, y1, x0, x1]
# DEM error - 2D
block = [box[1], box[3], box[0], box[2]]
writefile.write_hdf5_block(dem_err_file,
data=delta_z,
datasetName='dem',
block=block)
# corrected time-series - 3D
block = [0, num_date, box[1], box[3], box[0], box[2]]
writefile.write_hdf5_block(ts_cor_file,
data=ts_cor,
datasetName='timeseries',
block=block)
# residual time-series - 3D
block = [0, num_date, box[1], box[3], box[0], box[2]]
writefile.write_hdf5_block(ts_res_file,
data=ts_res,
datasetName='timeseries',
block=block)
# roll back to the origial number of threads
cluster.roll_back_num_threads(num_threads_dict)
# time info
m, s = divmod(time.time() - start_time, 60)
print('time used: {:02.0f} mins {:02.1f} secs.'.format(m, s))
return dem_err_file, ts_cor_file, ts_res_file
def write2hdf5(out_file, ts_file, coh_file, mask_file, geom_file, metadata):
"""Write HDF5 file in HDF-EOS5 format"""
ts_obj = timeseries(ts_file)
ts_obj.open(print_msg=False)
dateList = ts_obj.dateList
# Open HDF5 File
f = h5py.File(out_file, 'w')
print('create HDF5 file: {} with w mode'.format(out_file))
maxDigit = 20
# Write Observation - Displacement
gName = 'HDFEOS/GRIDS/timeseries/observation'
print('create group /{}'.format(gName))
group = f.create_group(gName)
dsName = 'displacement'
data = ts_obj.read(print_msg=False)
print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'
' with compression={c}').format(g=gName,
d=dsName,
w=maxDigit,
t=str(data.dtype),
s=data.shape,
c=compression))
dset = group.create_dataset(dsName,
data=data,
dtype=np.float32,
chunks=True,
compression=compression)
dset.attrs['Title'] = dsName
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = 'meters'
dsName = 'date'
data = np.array(dateList, dtype=np.string_)
group.create_dataset(dsName, data=data)
print('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'.format(g=gName,
d=dsName,
w=maxDigit,
t=str(data.dtype),
s=data.shape))
dsName = 'bperp'
data = np.array(ts_obj.pbase, dtype=np.float32)
group.create_dataset(dsName, data=data)
print('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'.format(g=gName,
d=dsName,
w=maxDigit,
t=str(data.dtype),
s=data.shape))
# Write Quality
gName = 'HDFEOS/GRIDS/timeseries/quality'
print('create group /{}'.format(gName))
group = f.create_group(gName)
## 1 - temporalCoherence
dsName = 'temporalCoherence'
data = readfile.read(coh_file)[0]
print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'
' with compression={c}').format(g=gName,
d=dsName,
w=maxDigit,
t=str(data.dtype),
s=data.shape,
c=compression))
dset = group.create_dataset(dsName,
data=data,
chunks=True,
compression=compression)
dset.attrs['Title'] = dsName
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = '1'
## 2 - mask
dsName = 'mask'
data = readfile.read(mask_file, datasetName='mask')[0]
print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'
' with compression={c}').format(g=gName,
d=dsName,
w=maxDigit,
t=str(data.dtype),
s=data.shape,
c=compression))
dset = group.create_dataset(dsName,
data=data,
chunks=True,
compression=compression)
dset.attrs['Title'] = dsName
dset.attrs['MissingValue'] = BOOL_ZERO
dset.attrs['_FillValue'] = BOOL_ZERO
dset.attrs['Units'] = '1'
# Write Geometry
# Required: height, incidenceAngle
# Optional: rangeCoord, azimuthCoord, azimuthAngle, slantRangeDistance, waterMask, shadowMask
gName = 'HDFEOS/GRIDS/timeseries/geometry'
print('create group /{}'.format(gName))
group = f.create_group(gName)
geom_obj = geometry(geom_file)
geom_obj.open(print_msg=False)
for dsName in geom_obj.datasetNames:
data = geom_obj.read(datasetName=dsName, print_msg=False)
print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'
' with compression={c}').format(g=gName,
d=dsName,
w=maxDigit,
t=str(data.dtype),
s=data.shape,
c=compression))
dset = group.create_dataset(dsName,
data=data,
chunks=True,
compression=compression)
dset.attrs['Title'] = dsName
if dsName in ['height',
'slantRangeDistance',
'bperp']:
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = 'meters'
elif dsName in ['incidenceAngle',
'azimuthAngle',
'latitude',
'longitude']:
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = 'degrees'
elif dsName in ['rangeCoord', 'azimuthCoord']:
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = '1'
elif dsName in ['waterMask', 'shadowMask']:
dset.attrs['MissingValue'] = BOOL_ZERO
dset.attrs['_FillValue'] = BOOL_ZERO
dset.attrs['Units'] = '1'
# Write Attributes to the HDF File
print('write metadata to root level')
for key, value in iter(metadata.items()):
f.attrs[key] = value
f.close()
print('finished writing to {}'.format(out_file))
return out_file
def write_hdf5_file(metadata, out_file, ts_file, tcoh_file, scoh_file,
mask_file, geom_file):
"""Write HDF5 file in HDF-EOS5 format."""
ts_obj = timeseries(ts_file)
ts_obj.open(print_msg=False)
dateList = ts_obj.dateList
numDate = len(dateList)
# Open HDF5 File
f = h5py.File(out_file, 'w')
print('create HDF5 file: {} with w mode'.format(out_file))
max_digit = 55
##### Group - Observation
gName = 'HDFEOS/GRIDS/timeseries/observation'
print('create group /{}'.format(gName))
group = f.create_group(gName)
## O1 - displacement
dsName = 'displacement'
dsShape = (numDate, ts_obj.length, ts_obj.width)
dsDataType = np.float32
print(('create dataset /{d:<{w}} of {t:<10} in size of {s}'
' with compression={c}').format(d='{}/{}'.format(gName, dsName),
w=max_digit,
t='float32',
s=dsShape,
c=COMPRESSION))
dset = group.create_dataset(dsName,
shape=dsShape,
maxshape=(None, dsShape[1], dsShape[2]),
dtype=dsDataType,
chunks=True,
compression=COMPRESSION)
print('write data acquition by acquition ...')
prog_bar = ptime.progressBar(maxValue=numDate)
for i in range(numDate):
dset[i, :, :] = readfile.read(ts_file, datasetName=dateList[i])[0]
prog_bar.update(i + 1,
suffix='{}/{} {}'.format(i + 1, numDate, dateList[i]))
prog_bar.close()
# attributes
dset.attrs['Title'] = dsName
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = 'meters'
## O2 - date
dsName = 'date'
data = np.array(dateList, dtype=np.string_)
dset = create_hdf5_dataset(group, dsName, data)
## O3 - perp baseline
dsName = 'bperp'
data = np.array(ts_obj.pbase, dtype=np.float32)
dset = create_hdf5_dataset(group, dsName, data)
##### Group - Quality
gName = 'HDFEOS/GRIDS/timeseries/quality'
print('create group /{}'.format(gName))
group = f.create_group(gName)
## Q1 - temporalCoherence
dsName = 'temporalCoherence'
# read
data = readfile.read(tcoh_file)[0]
# write
dset = create_hdf5_dataset(group, dsName, data)
# attributes
dset.attrs['Title'] = dsName
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = '1'
## Q2 - avgSpatialCoherence
dsName = 'avgSpatialCoherence'
# read
data = readfile.read(scoh_file)[0]
# write
dset = create_hdf5_dataset(group, dsName, data)
# attributes
dset.attrs['Title'] = dsName
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = '1'
## Q3 - mask
dsName = 'mask'
# read
data = readfile.read(mask_file, datasetName='mask')[0]
# write
dset = create_hdf5_dataset(group, dsName, data)
# attributes
dset.attrs['Title'] = dsName
dset.attrs['MissingValue'] = BOOL_ZERO
dset.attrs['_FillValue'] = BOOL_ZERO
dset.attrs['Units'] = '1'
##### Group - Write Geometry
# Required: height, incidenceAngle
# Optional: rangeCoord, azimuthCoord, azimuthAngle, slantRangeDistance, waterMask, shadowMask
gName = 'HDFEOS/GRIDS/timeseries/geometry'
print('create group /{}'.format(gName))
group = f.create_group(gName)
geom_obj = geometry(geom_file)
geom_obj.open(print_msg=False)
for dsName in geom_obj.datasetNames:
# read
data = geom_obj.read(datasetName=dsName, print_msg=False)
# write
dset = create_hdf5_dataset(group, dsName, data)
# attributes
dset.attrs['Title'] = dsName
if dsName in ['height', 'slantRangeDistance', 'bperp']:
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = 'meters'
elif dsName in [
'incidenceAngle', 'azimuthAngle', 'latitude', 'longitude'
]:
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = 'degrees'
elif dsName in ['rangeCoord', 'azimuthCoord']:
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = '1'
elif dsName in ['waterMask', 'shadowMask']:
dset.attrs['MissingValue'] = BOOL_ZERO
dset.attrs['_FillValue'] = BOOL_ZERO
dset.attrs['Units'] = '1'
# Write Attributes to the HDF File
print('write metadata to root level')
for key, value in iter(metadata.items()):
f.attrs[key] = value
f.close()
print('finished writing to {}'.format(out_file))
return out_file
def calc_delay_timeseries(inps):
"""Calculate delay time-series and write it to HDF5 file.
Parameters: inps : namespace, all input parameters
Returns: tropo_file : str, file name of ECMWF.h5
"""
def get_dataset_size(fname):
atr = readfile.read_attribute(fname)
shape = (int(atr['LENGTH']), int(atr['WIDTH']))
return shape
def run_or_skip(grib_files, tropo_file, geom_file):
print('update mode: ON')
print('output file: {}'.format(tropo_file))
flag = 'skip'
# check existance and modification time
if ut.run_or_skip(out_file=tropo_file,
in_file=grib_files,
print_msg=False) == 'run':
flag = 'run'
print(
'1) output file either do NOT exist or is NOT newer than all GRIB files.'
)
else:
print('1) output file exists and is newer than all GRIB files.')
# check dataset size in space / time
date_list = [
str(re.findall('\d{8}', os.path.basename(i))[0])
for i in grib_files
]
if (get_dataset_size(tropo_file) != get_dataset_size(geom_file)
or any(i not in timeseries(tropo_file).get_date_list()
for i in date_list)):
flag = 'run'
print(
'2) output file does NOT have the same len/wid as the geometry file {} or does NOT contain all dates'
.format(geom_file))
else:
print(
'2) output file has the same len/wid as the geometry file and contains all dates'
)
# check if output file is fully written
with h5py.File(tropo_file, 'r') as f:
if np.all(f['timeseries'][-1, :, :] == 0):
flag = 'run'
print('3) output file is NOT fully written.')
else:
print('3) output file is fully written.')
# result
print('run or skip: {}'.format(flag))
return flag
if run_or_skip(inps.grib_files, inps.tropo_file, inps.geom_file) == 'skip':
return
## 1. prepare geometry data
geom_obj = geometry(inps.geom_file)
geom_obj.open()
inps.inc = geom_obj.read(datasetName='incidenceAngle')
inps.dem = geom_obj.read(datasetName='height')
# for testing
if inps.custom_height:
print(
'use input custom height of {} m for vertical integration'.format(
inps.custom_height))
inps.dem[:] = inps.custom_height
if 'latitude' in geom_obj.datasetNames:
# for lookup table in radar-coord (isce, doris)
inps.lat = geom_obj.read(datasetName='latitude')
inps.lon = geom_obj.read(datasetName='longitude')
elif 'Y_FIRST' in geom_obj.metadata:
# for lookup table in geo-coded (gamma, roipac) and obs. in geo-coord
inps.lat, inps.lon = ut.get_lat_lon(geom_obj.metadata)
# convert coordinates to lat/lon, e.g. from UTM for ASF HyPP3
if not geom_obj.metadata['Y_UNIT'].startswith('deg'):
inps.lat, inps.lon = ut.to_latlon(inps.atr['OG_FILE_PATH'],
inps.lon, inps.lat)
else:
# for lookup table in geo-coded (gamma, roipac) and obs. in radar-coord
inps.lat, inps.lon = ut.get_lat_lon_rdc(inps.atr)
# mask of valid pixels
mask = np.multiply(inps.inc != 0, ~np.isnan(inps.inc))
## 2. prepare output file
# metadata
atr = inps.atr.copy()
atr['FILE_TYPE'] = 'timeseries'
atr['UNIT'] = 'm'
# remove metadata related with double reference
# because absolute delay is calculated and saved
for key in ['REF_DATE', 'REF_X', 'REF_Y', 'REF_LAT', 'REF_LON']:
if key in atr.keys():
atr.pop(key)
# instantiate time-series
length, width = int(atr['LENGTH']), int(atr['WIDTH'])
num_date = len(inps.grib_files)
date_list = [
str(re.findall('\d{8}', os.path.basename(i))[0])
for i in inps.grib_files
]
dates = np.array(date_list, dtype=np.string_)
ds_name_dict = {
"date": [dates.dtype, (num_date, ), dates],
"timeseries": [np.float32, (num_date, length, width), None],
}
writefile.layout_hdf5(inps.tropo_file, ds_name_dict, metadata=atr)
## 3. calculate phase delay
print(
'\n------------------------------------------------------------------------------'
)
print(
'calculating absolute delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...'
)
print('number of grib files used: {}'.format(num_date))
prog_bar = ptime.progressBar(maxValue=num_date, print_msg=~inps.verbose)
for i in range(num_date):
grib_file = inps.grib_files[i]
# calc tropo delay
tropo_data = get_delay(grib_file,
tropo_model=inps.tropo_model,
delay_type=inps.delay_type,
dem=inps.dem,
inc=inps.inc,
lat=inps.lat,
lon=inps.lon,
mask=mask,
verbose=inps.verbose)
# write tropo delay to file
block = [i, i + 1, 0, length, 0, width]
writefile.write_hdf5_block(inps.tropo_file,
data=tropo_data,
datasetName='timeseries',
block=block,
print_msg=False)
prog_bar.update(i + 1, suffix=os.path.basename(grib_file))
prog_bar.close()
return inps.tropo_file
def calculate_delay_timeseries(inps):
"""Calculate delay time-series and write it to HDF5 file.
Parameters: inps : namespace, all input parameters
Returns: tropo_file : str, file name of ECMWF.h5
"""
def get_dataset_size(fname):
atr = readfile.read_attribute(fname)
shape = (int(atr['LENGTH']), int(atr['WIDTH']))
return shape
# check existing tropo delay file
if (ut.run_or_skip(out_file=inps.tropo_file,
in_file=inps.grib_files,
print_msg=False) == 'skip'
and get_dataset_size(inps.tropo_file) == get_dataset_size(
inps.geom_file)):
print(
'{} file exists and is newer than all GRIB files, skip updating.'.
format(inps.tropo_file))
return
# prepare geometry data
geom_obj = geometry(inps.geom_file)
geom_obj.open()
inps.dem = geom_obj.read(datasetName='height')
inps.inc = geom_obj.read(datasetName='incidenceAngle')
if 'latitude' in geom_obj.datasetNames:
# for dataset in geo OR radar coord with lookup table in radar-coord (isce, doris)
inps.lat = geom_obj.read(datasetName='latitude')
inps.lon = geom_obj.read(datasetName='longitude')
elif 'Y_FIRST' in geom_obj.metadata:
# for geo-coded dataset (gamma, roipac)
inps.lat, inps.lon = ut.get_lat_lon(geom_obj.metadata)
else:
# for radar-coded dataset (gamma, roipac)
inps.lat, inps.lon = ut.get_lat_lon_rdc(geom_obj.metadata)
# calculate phase delay
length, width = int(inps.atr['LENGTH']), int(inps.atr['WIDTH'])
num_date = len(inps.grib_files)
date_list = [str(re.findall('\d{8}', i)[0]) for i in inps.grib_files]
tropo_data = np.zeros((num_date, length, width), np.float32)
print(
'\n------------------------------------------------------------------------------'
)
print(
'calcualting absolute delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...'
)
print('number of grib files used: {}'.format(num_date))
prog_bar = ptime.progressBar(maxValue=num_date)
for i in range(num_date):
grib_file = inps.grib_files[i]
tropo_data[i] = get_delay(grib_file, inps)
prog_bar.update(i + 1, suffix=os.path.basename(grib_file))
prog_bar.close()
# remove metadata related with double reference
# because absolute delay is calculated and saved
for key in ['REF_DATE', 'REF_X', 'REF_Y', 'REF_LAT', 'REF_LON']:
if key in inps.atr.keys():
inps.atr.pop(key)
# Write tropospheric delay to HDF5
ts_obj = timeseries(inps.tropo_file)
ts_obj.write2hdf5(data=tropo_data,
dates=date_list,
metadata=inps.atr,
refFile=inps.timeseries_file)
return inps.tropo_file
def check_loaded_dataset(work_dir='./', print_msg=True, relpath=False):
"""Check the result of loading data for the following two rules:
1. file existance
2. file attribute readability
Parameters: work_dir : string, MintPy working directory
print_msg : bool, print out message
Returns: True, if all required files and dataset exist; otherwise, ERROR
If True, PROCESS, SLC folder could be removed.
stack_file :
geom_file :
lookup_file :
Example: work_dir = os.path.expandvars('./FernandinaSenDT128/mintpy')
ut.check_loaded_dataset(work_dir)
"""
load_complete = True
if not work_dir:
work_dir = os.getcwd()
work_dir = os.path.abspath(work_dir)
# 1. interferograms stack file: unwrapPhase, coherence
ds_list = ['unwrapPhase', 'rangeOffset', 'azimuthOffset']
flist = [os.path.join(work_dir, 'inputs/ifgramStack.h5')]
stack_file = is_file_exist(flist, abspath=True)
if stack_file is not None:
obj = ifgramStack(stack_file)
obj.open(print_msg=False)
if all(x not in obj.datasetNames for x in ds_list):
msg = 'required dataset is missing in file {}:'.format(stack_file)
msg += '\n' + ' OR '.join(ds_list)
raise ValueError(msg)
# check coherence for phase stack
if 'unwrapPhase' in obj.datasetNames and 'coherence' not in obj.datasetNames:
print('WARNING: "coherence" is missing in file {}'.format(
stack_file))
else:
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
'./inputs/ifgramStack.h5')
atr = readfile.read_attribute(stack_file)
# 2. geom_file: height
if 'X_FIRST' in atr.keys():
flist = [os.path.join(work_dir, 'inputs/geometryGeo.h5')]
else:
flist = [os.path.join(work_dir, 'inputs/geometryRadar.h5')]
geom_file = is_file_exist(flist, abspath=True)
if geom_file is not None:
obj = geometry(geom_file)
obj.open(print_msg=False)
dname = geometryDatasetNames[0]
if dname not in obj.datasetNames:
raise ValueError(
'required dataset "{}" is missing in file {}'.format(
dname, geom_file))
else:
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
'./inputs/geometry*.h5')
# 3. lookup_file: latitude,longitude or rangeCoord,azimuthCoord
# could be different than geometry file in case of roipac and gamma
flist = [os.path.join(work_dir, 'inputs/geometry*.h5')]
lookup_file = get_lookup_file(flist, abspath=True, print_msg=print_msg)
if 'X_FIRST' not in atr.keys():
if lookup_file is not None:
obj = geometry(lookup_file)
obj.open(print_msg=False)
if atr['PROCESSOR'] in ['isce', 'doris']:
dnames = geometryDatasetNames[1:3]
elif atr['PROCESSOR'] in ['gamma', 'roipac']:
dnames = geometryDatasetNames[3:5]
else:
raise AttributeError('InSAR processor: {}'.format(
atr['PROCESSOR']))
for dname in dnames:
if dname not in obj.datasetNames:
load_complete = False
raise Exception(
'required dataset "{}" is missing in file {}'.format(
dname, lookup_file))
else:
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
'./inputs/geometry*.h5')
else:
print("Input data seems to be geocoded. Lookup file not needed.")
if relpath:
stack_file = os.path.relpath(stack_file) if stack_file else stack_file
geom_file = os.path.relpath(geom_file) if geom_file else geom_file
lookup_file = os.path.relpath(
lookup_file) if lookup_file else lookup_file
# print message
if print_msg:
print(('Loaded dataset are processed by '
'InSAR software: {}'.format(atr['PROCESSOR'])))
if 'X_FIRST' in atr.keys():
print('Loaded dataset is in GEO coordinates')
else:
print('Loaded dataset is in RADAR coordinates')
print('Interferograms Stack: {}'.format(stack_file))
print('Geometry File : {}'.format(geom_file))
print('Lookup Table File : {}'.format(lookup_file))
if load_complete:
print('-' * 50)
print(
'All data needed found/loaded/copied. Processed 2-pass InSAR data can be removed.'
)
print('-' * 50)
return load_complete, stack_file, geom_file, lookup_file
def write2hdf5(out_file, ts_file, coh_file, mask_file, geom_file, metadata):
"""Write HDF5 file in HDF-EOS5 format"""
ts_obj = timeseries(ts_file)
ts_obj.open(print_msg=False)
dateList = ts_obj.dateList
# Open HDF5 File
f = h5py.File(out_file, 'w')
print('create HDF5 file: {} with w mode'.format(out_file))
maxDigit = 20
# Write Observation - Displacement
gName = 'HDFEOS/GRIDS/timeseries/observation'
print('create group /{}'.format(gName))
group = f.create_group(gName)
dsName = 'displacement'
data = ts_obj.read(print_msg=False)
print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'
' with compression={c}').format(g=gName,
d=dsName,
w=maxDigit,
t=str(data.dtype),
s=data.shape,
c=compression))
dset = group.create_dataset(dsName,
data=data,
dtype=np.float32,
chunks=True,
compression=compression)
dset.attrs['Title'] = dsName
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = 'meters'
dsName = 'date'
data = np.array(dateList, dtype=np.string_)
group.create_dataset(dsName, data=data)
print('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'.format(
g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape))
dsName = 'bperp'
data = np.array(ts_obj.pbase, dtype=np.float32)
group.create_dataset(dsName, data=data)
print('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'.format(
g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape))
# Write Quality
gName = 'HDFEOS/GRIDS/timeseries/quality'
print('create group /{}'.format(gName))
group = f.create_group(gName)
## 1 - temporalCoherence
dsName = 'temporalCoherence'
data = readfile.read(coh_file)[0]
print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'
' with compression={c}').format(g=gName,
d=dsName,
w=maxDigit,
t=str(data.dtype),
s=data.shape,
c=compression))
dset = group.create_dataset(dsName,
data=data,
chunks=True,
compression=compression)
dset.attrs['Title'] = dsName
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = '1'
## 2 - mask
dsName = 'mask'
data = readfile.read(mask_file, datasetName='mask')[0]
print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'
' with compression={c}').format(g=gName,
d=dsName,
w=maxDigit,
t=str(data.dtype),
s=data.shape,
c=compression))
dset = group.create_dataset(dsName,
data=data,
chunks=True,
compression=compression)
dset.attrs['Title'] = dsName
dset.attrs['MissingValue'] = BOOL_ZERO
dset.attrs['_FillValue'] = BOOL_ZERO
dset.attrs['Units'] = '1'
# Write Geometry
# Required: height, incidenceAngle
# Optional: rangeCoord, azimuthCoord, azimuthAngle, slantRangeDistance, waterMask, shadowMask
gName = 'HDFEOS/GRIDS/timeseries/geometry'
print('create group /{}'.format(gName))
group = f.create_group(gName)
geom_obj = geometry(geom_file)
geom_obj.open(print_msg=False)
for dsName in geom_obj.datasetNames:
data = geom_obj.read(datasetName=dsName, print_msg=False)
print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'
' with compression={c}').format(g=gName,
d=dsName,
w=maxDigit,
t=str(data.dtype),
s=data.shape,
c=compression))
dset = group.create_dataset(dsName,
data=data,
chunks=True,
compression=compression)
dset.attrs['Title'] = dsName
if dsName in ['height', 'slantRangeDistance', 'bperp']:
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = 'meters'
elif dsName in [
'incidenceAngle', 'azimuthAngle', 'latitude', 'longitude'
]:
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = 'degrees'
elif dsName in ['rangeCoord', 'azimuthCoord']:
dset.attrs['MissingValue'] = FLOAT_ZERO
dset.attrs['_FillValue'] = FLOAT_ZERO
dset.attrs['Units'] = '1'
elif dsName in ['waterMask', 'shadowMask']:
dset.attrs['MissingValue'] = BOOL_ZERO
dset.attrs['_FillValue'] = BOOL_ZERO
dset.attrs['Units'] = '1'
# Write Attributes to the HDF File
print('write metadata to root level')
for key, value in iter(metadata.items()):
f.attrs[key] = value
f.close()
print('finished writing to {}'.format(out_file))
return out_file
