def run_geocode(inps):
"""geocode all input files"""
start_time = time.time()
# Prepare geometry for geocoding
res_obj = resample(lookupFile=inps.lookupFile,
dataFile=inps.file[0],
SNWE=inps.SNWE,
laloStep=inps.laloStep,
processor=inps.processor)
res_obj.open()
# resample input files one by one
for infile in inps.file:
print('-' * 50+'\nresampling file: {}'.format(infile))
ext = os.path.splitext(infile)[1]
atr = readfile.read_attribute(infile, datasetName=inps.dset)
outfile = auto_output_filename(infile, inps)
if inps.updateMode and ut.run_or_skip(outfile, in_file=[infile, inps.lookupFile]) == 'skip':
print('update mode is ON, skip geocoding.')
continue
# read source data and resample
dsNames = readfile.get_dataset_list(infile, datasetName=inps.dset)
maxDigit = max([len(i) for i in dsNames])
dsResDict = dict()
for dsName in dsNames:
print('reading {d:<{w}} from {f} ...'.format(d=dsName,
w=maxDigit,
f=os.path.basename(infile)))
if ext in ['.h5','.he5']:
data = readfile.read(infile, datasetName=dsName, print_msg=False)[0]
else:
data, atr = readfile.read(infile, datasetName=dsName, print_msg=False)
# keep timeseries data as 3D matrix when there is only one acquisition
# because readfile.read() will squeeze it to 2D
if atr['FILE_TYPE'] == 'timeseries' and len(data.shape) == 2:
data = np.reshape(data, (1, data.shape[0], data.shape[1]))
res_data = res_obj.run_resample(src_data=data,
interp_method=inps.interpMethod,
fill_value=inps.fillValue,
nprocs=inps.nprocs,
print_msg=True)
dsResDict[dsName] = res_data
# update metadata
if inps.radar2geo:
atr = metadata_radar2geo(atr, res_obj)
else:
atr = metadata_geo2radar(atr, res_obj)
#if len(dsNames) == 1 and dsName not in ['timeseries']:
# atr['FILE_TYPE'] = dsNames[0]
# infile = None
writefile.write(dsResDict, out_file=outfile, metadata=atr, ref_file=infile)
m, s = divmod(time.time()-start_time, 60)
print('time used: {:02.0f} mins {:02.1f} secs.\n'.format(m, s))
return outfile
def run_geocode(inps):
"""geocode all input files"""
start_time = time.time()
# feed the largest file for resample object initiation
ind_max = np.argmax([os.path.getsize(i) for i in inps.file])
# prepare geometry for geocoding
kwargs = dict(interp_method=inps.interpMethod,
fill_value=inps.fillValue,
nprocs=inps.nprocs,
max_memory=inps.maxMemory,
software=inps.software,
print_msg=True)
if inps.latFile and inps.lonFile:
kwargs['lat_file'] = inps.latFile
kwargs['lon_file'] = inps.lonFile
res_obj = resample(lut_file=inps.lookupFile,
src_file=inps.file[ind_max],
SNWE=inps.SNWE,
lalo_step=inps.laloStep,
**kwargs)
res_obj.open()
res_obj.prepare()
# resample input files one by one
for infile in inps.file:
print('-' * 50 + '\nresampling file: {}'.format(infile))
ext = os.path.splitext(infile)[1]
atr = readfile.read_attribute(infile, datasetName=inps.dset)
outfile = auto_output_filename(infile, inps)
# update_mode
if inps.updateMode:
print('update mode: ON')
if ut.run_or_skip(outfile, in_file=[infile,
inps.lookupFile]) == 'skip':
continue
## prepare output
# update metadata
if inps.radar2geo:
atr = attr.update_attribute4radar2geo(atr, res_obj=res_obj)
else:
atr = attr.update_attribute4geo2radar(atr, res_obj=res_obj)
# instantiate output file
file_is_hdf5 = os.path.splitext(infile)[1] in ['.h5', '.he5']
if file_is_hdf5:
writefile.layout_hdf5(outfile, metadata=atr, ref_file=infile)
else:
dsDict = dict()
## run
dsNames = readfile.get_dataset_list(infile, datasetName=inps.dset)
maxDigit = max([len(i) for i in dsNames])
for dsName in dsNames:
if not file_is_hdf5:
dsDict[dsName] = np.zeros((res_obj.length, res_obj.width))
# loop for block-by-block IO
for i in range(res_obj.num_box):
src_box = res_obj.src_box_list[i]
dest_box = res_obj.dest_box_list[i]
# read
print('-' * 50 +
'\nreading {d:<{w}} in block {b} from {f} ...'.format(
d=dsName,
w=maxDigit,
b=src_box,
f=os.path.basename(infile)))
data = readfile.read(infile,
datasetName=dsName,
box=src_box,
print_msg=False)[0]
# resample
data = res_obj.run_resample(src_data=data, box_ind=i)
# write / save block data
if data.ndim == 3:
block = [
0, data.shape[0], dest_box[1], dest_box[3],
dest_box[0], dest_box[2]
]
else:
block = [
dest_box[1], dest_box[3], dest_box[0], dest_box[2]
]
if file_is_hdf5:
print('write data in block {} to file: {}'.format(
block, outfile))
writefile.write_hdf5_block(outfile,
data=data,
datasetName=dsName,
block=block,
print_msg=False)
else:
dsDict[dsName][block[0]:block[1], block[2]:block[3]] = data
# for binary file: ensure same data type
if not file_is_hdf5:
dsDict[dsName] = np.array(dsDict[dsName], dtype=data.dtype)
# write binary file
if not file_is_hdf5:
writefile.write(dsDict,
out_file=outfile,
metadata=atr,
ref_file=infile)
m, s = divmod(time.time() - start_time, 60)
print('time used: {:02.0f} mins {:02.1f} secs.\n'.format(m, s))
return outfile
def calc_solid_earth_tides_timeseries(ts_file, geom_file, set_file, date_wise_acq_time=False,
update_mode=True, verbose=False):
"""Calculate the time-series of solid Earth tides (SET) in LOS direction.
Parameters: ts_file - str, path of the time-series HDF5 file
geom_file - str, path of the geometry HDF5 file
set_file - str, output SET time-sereis file
date_wise_acq_time - bool, use the exact date-wise acquisition time
Returns: set_file - str, output SET time-sereis file
"""
if update_mode and os.path.isfile(set_file):
print('update mode: ON')
print('skip re-calculating and use existing file: {}'.format(set_file))
return set_file
# prepare LOS geometry: geocoding if in radar-coordinates
inc_angle, head_angle, atr_geo = prepare_los_geometry(geom_file)
# get LOS unit vector
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=RuntimeWarning)
unit_vec = [
np.sin(inc_angle) * np.cos(head_angle) * -1,
np.sin(inc_angle) * np.sin(head_angle),
np.cos(inc_angle),
]
# prepare datetime
dt_objs = get_datetime_list(ts_file, date_wise_acq_time=date_wise_acq_time)
# initiate data matrix
num_date = len(dt_objs)
length = int(atr_geo['LENGTH'])
width = int(atr_geo['WIDTH'])
ts_tide = np.zeros((num_date, length, width), dtype=np.float32)
# loop for calc
print('\n'+'-'*50)
print('calculating solid Earth tides using solid.for (D. Milbert, 2018) ...')
prog_bar = ptime.progressBar(maxValue=num_date, print_msg=not verbose)
for i, dt_obj in enumerate(dt_objs):
# calculate tide in ENU direction
(tide_e,
tide_n,
tide_u) = pysolid.calc_solid_earth_tides_grid(dt_obj, atr_geo,
display=False,
verbose=verbose)
# convert ENU to LOS direction
# sign convention: positive for motion towards satellite
ts_tide[i,:,:] = (tide_e * unit_vec[0]
+ tide_n * unit_vec[1]
+ tide_u * unit_vec[2])
prog_bar.update(i+1, suffix='{} ({}/{})'.format(dt_obj.isoformat(), i+1, num_date))
prog_bar.close()
# radar-coding if input in radar-coordinates
# use ts_file to avoid potential missing CENTER_LINE_UTC attributes in geom_file from alosStack
atr = readfile.read_attribute(ts_file)
if 'Y_FIRST' not in atr.keys():
print('radar-coding the LOS tides time-series ...')
res_obj = resample(lut_file=geom_file)
res_obj.open()
res_obj.src_meta = atr_geo
res_obj.prepare()
# resample data
box = res_obj.src_box_list[0]
ts_tide = res_obj.run_resample(src_data=ts_tide[:,
box[1]:box[3],
box[0]:box[2]])
## output
# attribute
atr['FILE_TYPE'] = 'timeseries'
atr['UNIT'] = 'm'
for key in ['REF_Y', 'REF_X', 'REF_DATE']:
if key in atr.keys():
atr.pop(key)
# write
ds_dict = {}
ds_dict['timeseries'] = ts_tide
ds_dict['sensingMid'] = np.array([i.strftime('%Y%m%dT%H%M%S') for i in dt_objs], dtype=np.string_)
writefile.write(ds_dict, out_file=set_file, metadata=atr, ref_file=ts_file)
return set_file
def prepare_los_geometry(geom_file):
"""Prepare LOS geometry data/info in geo-coordinates
Parameters: geom_file - str, path of geometry file
Returns: inc_angle - 2D np.ndarray, incidence angle in radians
head_angle - 2D np.ndarray, heading angle in radians
atr - dict, metadata in geo-coordinate
"""
print('prepare LOS geometry in geo-coordinates from file: {}'.format(geom_file))
atr = readfile.read_attribute(geom_file)
print('read incidenceAngle from file: {}'.format(geom_file))
inc_angle = readfile.read(geom_file, datasetName='incidenceAngle')[0]
if 'azimuthAngle' in readfile.get_dataset_list(geom_file):
print('read azimuthAngle from file: {}'.format(geom_file))
print('convert azimuth angle to heading angle')
az_angle = readfile.read(geom_file, datasetName='azimuthAngle')[0]
head_angle = ut.azimuth2heading_angle(az_angle)
else:
print('use the HEADING attribute as the mean heading angle')
head_angle = np.ones(inc_angle.shape, dtype=np.float32) * float(atr['HEADING'])
# geocode inc/az angle data if in radar-coord
if 'Y_FIRST' not in atr.keys():
print('-'*50)
print('geocoding the incidence / heading angles ...')
res_obj = resample(lut_file=geom_file, src_file=geom_file)
res_obj.open()
res_obj.prepare()
# resample data
box = res_obj.src_box_list[0]
inc_angle = res_obj.run_resample(src_data=inc_angle[box[1]:box[3], box[0]:box[2]])
head_angle = res_obj.run_resample(src_data=head_angle[box[1]:box[3], box[0]:box[2]])
# update attribute
atr = attr.update_attribute4radar2geo(atr, res_obj=res_obj)
# for 'Y_FIRST' not in 'degree'
# e.g. meters for UTM projection from ASF HyP3
if not atr['Y_UNIT'].lower().startswith('deg'):
# get SNWE in meter
length, width = int(atr['LENGTH']), int(atr['WIDTH'])
N = float(atr['Y_FIRST'])
W = float(atr['X_FIRST'])
y_step = float(atr['Y_STEP'])
x_step = float(atr['X_STEP'])
S = N + y_step * length
E = W + x_step * width
# SNWE in meter --> degree
lat0, lon0 = ut.to_latlon(atr['OG_FILE_PATH'], W, N)
lat1, lon1 = ut.to_latlon(atr['OG_FILE_PATH'], E, S)
lat_step = (lat1 - lat0) / length
lon_step = (lon1 - lon0) / width
# update Y/X_FIRST/STEP/UNIT
atr['Y_FIRST'] = lat0
atr['X_FIRST'] = lon0
atr['Y_STEP'] = lat_step
atr['X_STEP'] = lon_step
atr['Y_UNIT'] = 'degrees'
atr['X_UNIT'] = 'degrees'
# unit: degree to radian
inc_angle *= np.pi / 180.
head_angle *= np.pi / 180.
return inc_angle, head_angle, atr
def run_geocode(inps):
"""geocode all input files"""
start_time = time.time()
# Prepare geometry for geocoding
res_obj = resample(lookupFile=inps.lookupFile,
dataFile=inps.file[0],
SNWE=inps.SNWE,
laloStep=inps.laloStep,
processor=inps.processor)
res_obj.open()
# resample input files one by one
for infile in inps.file:
print('-' * 50 + '\nresampling file: {}'.format(infile))
ext = os.path.splitext(infile)[1]
atr = readfile.read_attribute(infile, datasetName=inps.dset)
outfile = auto_output_filename(infile, inps)
if inps.updateMode and ut.run_or_skip(
outfile, in_file=[infile, inps.lookupFile]) == 'skip':
print('update mode is ON, skip geocoding.')
continue
# read source data and resample
dsNames = readfile.get_dataset_list(infile, datasetName=inps.dset)
maxDigit = max([len(i) for i in dsNames])
dsResDict = dict()
for dsName in dsNames:
print('reading {d:<{w}} from {f} ...'.format(
d=dsName, w=maxDigit, f=os.path.basename(infile)))
if ext in ['.h5', '.he5']:
data = readfile.read(infile,
datasetName=dsName,
print_msg=False)[0]
else:
data, atr = readfile.read(infile,
datasetName=dsName,
print_msg=False)
# keep timeseries data as 3D matrix when there is only one acquisition
# because readfile.read() will squeeze it to 2D
if atr['FILE_TYPE'] == 'timeseries' and len(data.shape) == 2:
data = np.reshape(data, (1, data.shape[0], data.shape[1]))
res_data = res_obj.run_resample(src_data=data,
interp_method=inps.interpMethod,
fill_value=inps.fillValue,
nprocs=inps.nprocs,
print_msg=True)
dsResDict[dsName] = res_data
# update metadata
if inps.radar2geo:
atr = metadata_radar2geo(atr, res_obj)
else:
atr = metadata_geo2radar(atr, res_obj)
#if len(dsNames) == 1 and dsName not in ['timeseries']:
# atr['FILE_TYPE'] = dsNames[0]
# infile = None
writefile.write(dsResDict,
out_file=outfile,
metadata=atr,
ref_file=infile)
m, s = divmod(time.time() - start_time, 60)
print('time used: {:02.0f} mins {:02.1f} secs.\n'.format(m, s))
return outfile
def calc_solid_earth_tides_timeseries(date_list,
geom_file,
out_file,
update_mode=True,
verbose=False):
"""Calculate the time-series of solid Earth tides in LOS direction.
Parameters: date_list - list of str, dates in YYYYMMDD
geom_file - str, path of geometry file in geo coordinates
out_file - str, output time-sereis file
Returns: out_file - str, output time-sereis file
"""
if update_mode and os.path.isfile(out_file):
print('update mode: ON')
print('skip re-calculating and use existing file: {}'.format(out_file))
return out_file
# prepare LOS geometry: geocoding if in radar-coordinates
inc_angle, head_angle, atr_geo = prepare_los_geometry(geom_file)
# get LOS unit vector
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=RuntimeWarning)
unit_vec = [
np.sin(inc_angle) * np.cos(head_angle) * -1,
np.sin(inc_angle) * np.sin(head_angle),
np.cos(inc_angle),
]
# initiate data matrix
num_date = len(date_list)
length = int(atr_geo['LENGTH'])
width = int(atr_geo['WIDTH'])
ts_tide = np.zeros((num_date, length, width), dtype=np.float32)
# loop for calc
print('\n' + '-' * 50)
print(
'calculating solid Earth tides using solid.for (D. Milbert, 2018) ...')
prog_bar = ptime.progressBar(maxValue=num_date, print_msg=not verbose)
for i in range(num_date):
date_str = date_list[i]
# calculate tide in ENU direction
(tide_e, tide_n,
tide_u) = pysolid.calc_solid_earth_tides_grid(date_str,
atr_geo,
display=False,
verbose=verbose)
# convert ENU to LOS direction
# sign convention: positive for motion towards satellite
ts_tide[i, :, :] = (tide_e * unit_vec[0] + tide_n * unit_vec[1] +
tide_u * unit_vec[2])
prog_bar.update(i + 1,
suffix='{} ({}/{})'.format(date_list[i], i + 1,
num_date))
prog_bar.close()
# radar-coding if input in radar-coordinates
atr = readfile.read_attribute(geom_file)
if 'Y_FIRST' not in atr.keys():
print('radar-coding the LOS tides time-series ...')
res_obj = resample(lut_file=geom_file)
res_obj.open()
res_obj.src_meta = atr_geo
res_obj.prepare()
# resample data
box = res_obj.src_box_list[0]
ts_tide = res_obj.run_resample(src_data=ts_tide[:, box[1]:box[3],
box[0]:box[2]])
## output
# attribute
atr['FILE_TYPE'] = 'timeseries'
atr['UNIT'] = 'm'
for key in ['REF_Y', 'REF_X', 'REF_DATE']:
if key in atr.keys():
atr.pop(key)
# write
ds_dict = {}
ds_dict['date'] = np.array(date_list, dtype=np.string_)
ds_dict['timeseries'] = ts_tide
writefile.write(ds_dict, out_file=out_file, metadata=atr)
return out_file