def generate_temporal_coherence_mask(self):
"""Generate reliable pixel mask from temporal coherence"""
geom_file = ut.check_loaded_dataset(self.workDir, print_msg=False)[2]
tcoh_file = 'temporalCoherence.h5'
mask_file = 'maskTempCoh.h5'
tcoh_min = self.template['mintpy.networkInversion.minTempCoh']
scp_args = '{} -m {} -o {} --shadow {}'.format(tcoh_file, tcoh_min,
mask_file, geom_file)
print('generate_mask.py', scp_args)
# update mode: run only if:
# 1) output file exists and newer than input file, AND
# 2) all config keys are the same
config_keys = ['mintpy.networkInversion.minTempCoh']
print('update mode: ON')
flag = 'skip'
if ut.run_or_skip(out_file=mask_file,
in_file=tcoh_file,
print_msg=False) == 'run':
flag = 'run'
else:
print(
'1) output file: {} already exists and newer than input file: {}'
.format(mask_file, tcoh_file))
atr = readfile.read_attribute(mask_file)
if any(
str(self.template[i]) != atr.get(i, 'False')
for i in config_keys):
flag = 'run'
print(
'2) NOT all key configration parameters are the same: {}'.
format(config_keys))
else:
print('2) all key configuration parameters are the same: {}'.
format(config_keys))
print('run or skip: {}'.format(flag))
if flag == 'run':
mintpy.generate_mask.main(scp_args.split())
# update configKeys
atr = {}
for key in config_keys:
atr[key] = self.template[key]
ut.add_attribute(mask_file, atr)
# check number of pixels selected in mask file for following analysis
num_pixel = np.sum(readfile.read(mask_file)[0] != 0.)
print('number of reliable pixels: {}'.format(num_pixel))
min_num_pixel = float(
self.template['mintpy.networkInversion.minNumPixel'])
if num_pixel < min_num_pixel:
msg = "Not enough reliable pixels (minimum of {}). ".format(
int(min_num_pixel))
msg += "Try the following:\n"
msg += "1) Check the reference pixel and make sure it's not in areas with unwrapping errors\n"
msg += "2) Check the network and make sure it's fully connected without subsets"
raise RuntimeError(msg)
return
def main(iargs=None):
# check inputs
inps = cmd_line_parse(iargs)
# update mode
if inps.update_mode and run_or_skip(inps) == 'skip':
return inps.ifgram_file
start_time = time.time()
# run bridging
run_unwrap_error_bridge(inps.ifgram_file,
water_mask_file=inps.waterMaskFile,
ramp_type=inps.ramp,
radius=inps.bridgePtsRadius,
dsNameIn=inps.datasetNameIn,
dsNameOut=inps.datasetNameOut)
# config parameter
if os.path.splitext(inps.ifgram_file)[1] in ['.h5', '.he5']:
print('add/update the following configuration metadata to file:')
config_metadata = dict()
for key in configKeys:
config_metadata[key_prefix+key] = str(vars(inps)[key])
ut.add_attribute(inps.ifgram_file, config_metadata, print_msg=True)
m, s = divmod(time.time()-start_time, 60)
print('\ntime used: {:02.0f} mins {:02.1f} secs\nDone.'.format(m, s))
return inps.ifgram_file
def run_load_data(self, step_name):
"""Load InSAR stacks into HDF5 files in ./inputs folder.
It 1) copy auxiliary files into work directory (for Unvi of Miami only)
2) load all interferograms stack files into mintpy/inputs directory.
3) check loading result
4) add custom metadata (optional, for HDF-EOS5 format only)
"""
# 1) copy aux files (optional)
self._copy_aux_file()
# 2) loading data
scp_args = '--template {}'.format(self.templateFile)
if self.customTemplateFile:
scp_args += ' {}'.format(self.customTemplateFile)
if self.projectName:
scp_args += ' --project {}'.format(self.projectName)
# run
print("load_data.py", scp_args)
mintpy.load_data.main(scp_args.split())
os.chdir(self.workDir)
# 3) check loading result
load_complete, stack_file, geom_file = ut.check_loaded_dataset(self.workDir, print_msg=True)[0:3]
# 3.1) output waterMask.h5
water_mask_file = 'waterMask.h5'
if 'waterMask' in readfile.get_dataset_list(geom_file):
print('generate {} from {} for conveniency'.format(water_mask_file, geom_file))
if ut.run_or_skip(out_file=water_mask_file, in_file=geom_file) == 'run':
water_mask, atr = readfile.read(geom_file, datasetName='waterMask')
atr['FILE_TYPE'] = 'waterMask'
writefile.write(water_mask, out_file=water_mask_file, metadata=atr)
# 4) add custom metadata (optional)
if self.customTemplateFile:
print('updating {}, {} metadata based on custom template file: {}'.format(
os.path.basename(stack_file),
os.path.basename(geom_file),
os.path.basename(self.customTemplateFile)))
# use ut.add_attribute() instead of add_attribute.py because of
# better control of special metadata, such as SUBSET_X/YMIN
ut.add_attribute(stack_file, self.customTemplate)
ut.add_attribute(geom_file, self.customTemplate)
# 5) if not load_complete, plot and raise exception
if not load_complete:
# plot result if error occured
self.plot_result(print_aux=False, plot=plot)
# go back to original directory
print('Go back to directory:', self.cwd)
os.chdir(self.cwd)
# raise error
msg = 'step {}: NOT all required dataset found, exit.'.format(step_name)
raise RuntimeError(msg)
return
def run_load_data(self, step_name):
"""Load InSAR stacks into HDF5 files in ./inputs folder.
It 1) copy auxiliary files into work directory (for Unvi of Miami only)
2) load all interferograms stack files into mintpy/inputs directory.
3) check loading result
4) add custom metadata (optional, for HDF-EOS5 format only)
"""
# 1) copy aux files (optional)
self._copy_aux_file()
# 2) loading data
# compose list of input arguments
# instead of using command line then split
# to support path with whitespace
iargs = ['--template', self.templateFile]
if self.customTemplateFile:
iargs += [self.customTemplateFile]
if self.projectName:
iargs += ['--project', self.projectName]
# run command line
print('\nload_data.py', ' '.join(iargs))
mintpy.load_data.main(iargs)
# come back to working directory
os.chdir(self.workDir)
# 3) check loading result
load_complete, stack_file, geom_file = ut.check_loaded_dataset(self.workDir, print_msg=True)[0:3]
# 4) add custom metadata (optional)
if self.customTemplateFile:
print('updating {}, {} metadata based on custom template file: {}'.format(
os.path.basename(stack_file),
os.path.basename(geom_file),
os.path.basename(self.customTemplateFile)))
# use ut.add_attribute() instead of add_attribute.py because of
# better control of special metadata, such as SUBSET_X/YMIN
ut.add_attribute(stack_file, self.customTemplate)
ut.add_attribute(geom_file, self.customTemplate)
# 5) if not load_complete, plot and raise exception
if not load_complete:
# plot result if error occured
self.plot_result(print_aux=False, plot=plot)
# go back to original directory
print('Go back to directory:', self.cwd)
os.chdir(self.cwd)
# raise error
msg = 'step {}: NOT all required dataset found, exit.'.format(step_name)
raise RuntimeError(msg)
return
def get_phase_linking_coherence_mask(template, work_dir):
"""
Generate reliable pixel mask from temporal coherence
functions = [generate_mask, readfile, run_or_skip, add_attribute]
# from mintpy import generate_mask
# from mintpy.utils import readfile
# from mintpy.utils.utils import run_or_skip, add_attribute
"""
tcoh_file = os.path.join(work_dir, 'temporalCoherence.h5')
mask_file = os.path.join(work_dir, 'maskTempCoh.h5')
tcoh_min = float(template['minopy.timeseries.minTempCoh'])
scp_args = '{} -m {} --nonzero -o {} --update'.format(
tcoh_file, tcoh_min, mask_file)
print('generate_mask.py', scp_args)
# update mode: run only if:
# 1) output file exists and newer than input file, AND
# 2) all config keys are the same
print('update mode: ON')
flag = 'skip'
if ut.run_or_skip(out_file=mask_file, in_file=tcoh_file,
print_msg=False) == 'run':
flag = 'run'
print('run or skip: {}'.format(flag))
if flag == 'run':
generate_mask.main(scp_args.split())
# update configKeys
atr = {}
atr['minopy.timeseries.minTempCoh'] = tcoh_min
ut.add_attribute(mask_file, atr)
ut.add_attribute(mask_file, atr)
# check number of pixels selected in mask file for following analysis
#num_pixel = np.sum(readfile.read(mask_file)[0] != 0.)
#print('number of reliable pixels: {}'.format(num_pixel))
#min_num_pixel = float(template['mintpy.networkInversion.minNumPixel']) # 100
#if num_pixel < min_num_pixel:
# msg = "Not enough reliable pixels (minimum of {}). ".format(int(min_num_pixel))
# msg += "Try the following:\n"
# msg += "1) Check the reference pixel and make sure it's not in areas with unwrapping errors\n"
# msg += "2) Check the network and make sure it's fully connected without subsets"
# raise RuntimeError(msg)
return
def generate_temporal_coherence_mask(self):
"""Generate reliable pixel mask from temporal coherence"""
geom_file = ut.check_loaded_dataset(self.workDir, print_msg=False)[2]
tcoh_file = 'temporalCoherence.h5'
mask_file = 'maskTempCoh.h5'
tcoh_min = self.template['mintpy.networkInversion.minTempCoh']
scp_args = '{} -m {} -o {} --shadow {}'.format(tcoh_file, tcoh_min, mask_file, geom_file)
print('generate_mask.py', scp_args)
# update mode: run only if:
# 1) output file exists and newer than input file, AND
# 2) all config keys are the same
config_keys = ['mintpy.networkInversion.minTempCoh']
print('update mode: ON')
flag = 'skip'
if ut.run_or_skip(out_file=mask_file, in_file=tcoh_file, print_msg=False) == 'run':
flag = 'run'
else:
print('1) output file: {} already exists and newer than input file: {}'.format(mask_file, tcoh_file))
atr = readfile.read_attribute(mask_file)
if any(str(self.template[i]) != atr.get(i, 'False') for i in config_keys):
flag = 'run'
print('2) NOT all key configration parameters are the same: {}'.format(config_keys))
else:
print('2) all key configuration parameters are the same: {}'.format(config_keys))
print('run or skip: {}'.format(flag))
if flag == 'run':
mintpy.generate_mask.main(scp_args.split())
# update configKeys
atr = {}
for key in config_keys:
atr[key] = self.template[key]
ut.add_attribute(mask_file, atr)
# check number of pixels selected in mask file for following analysis
num_pixel = np.sum(readfile.read(mask_file)[0] != 0.)
print('number of reliable pixels: {}'.format(num_pixel))
min_num_pixel = float(self.template['mintpy.networkInversion.minNumPixel'])
if num_pixel < min_num_pixel:
msg = "Not enough reliable pixels (minimum of {}). ".format(int(min_num_pixel))
msg += "Try the following:\n"
msg += "1) Check the reference pixel and make sure it's not in areas with unwrapping errors\n"
msg += "2) Check the network and make sure it's fully connected without subsets"
raise RuntimeError(msg)
return
def update_file_attribute(fname, atr_new):
# Read Original Attributes
atr = readfile.read_attribute(fname)
print('update {} file attribute: {}'.format(atr['FILE_TYPE'], fname))
ext = os.path.splitext(fname)[1]
if ext in ['.h5', '.he5']:
fname = ut.add_attribute(fname, atr_new)
else:
if not ut.update_attribute_or_not(atr_new, atr):
print(
'All updated (removed) attributes already exists (do not exists) and have the same value, skip update.'
)
else:
for key, value in iter(atr_new.items()):
if value == 'None':
try:
atr.pop(key)
except:
pass
else:
atr[key] = value
rsc_file = '{}.rsc'.format(fname)
print('writing >>> {}'.format(rsc_file))
writefile.write_roipac_rsc(atr, out_file=rsc_file)
return fname
def remove_reference_pixel(File):
"""Remove reference pixel info from input file"""
print("remove REF_Y/X and/or REF_LAT/LON from file: "+File)
atrDrop = {}
for i in ['REF_X', 'REF_Y', 'REF_LAT', 'REF_LON']:
atrDrop[i] = 'None'
File = ut.add_attribute(File, atrDrop)
return File
def remove_reference_pixel(File):
"""Remove reference pixel info from input file"""
print("remove REF_Y/X and/or REF_LAT/LON from file: " + File)
atrDrop = {}
for i in ['REF_X', 'REF_Y', 'REF_LAT', 'REF_LON']:
atrDrop[i] = 'None'
File = ut.add_attribute(File, atrDrop)
return File
def run_save2hdfeos5(self, step_name):
"""Save displacement time-series and its aux data in geo coordinate into HDF-EOS5 format"""
if self.template['mintpy.save.hdfEos5'] is True:
# input
ts_file = self.get_timeseries_filename(
self.template)[step_name]['input']
# Add attributes from custom template to timeseries file
if self.customTemplate is not None:
ut.add_attribute(ts_file, self.customTemplate)
tcoh_file = 'temporalCoherence.h5'
mask_file = 'geo_maskTempCoh.h5'
geom_file = ut.check_loaded_dataset(self.workDir,
print_msg=False)[2]
if 'geo' in ts_file:
tcoh_file = './geo/geo_temporalCoherence.h5'
mask_file = './geo/geo_maskTempCoh.h5'
geom_file = './geo/geo_{}'.format(os.path.basename(geom_file))
# cmd
print('--------------------------------------------')
scp_args = '{f} -c {c} -m {m} -g {g} -t {t}'.format(
f=ts_file,
c=tcoh_file,
m=mask_file,
g=geom_file,
t=self.templateFile)
print('save_hdfeos5.py', scp_args)
# output (check existing file)
atr = readfile.read_attribute(ts_file)
SAT = sensor.get_unavco_mission_name(atr)
try:
hdfeos5_file = get_file_list('{}_*.he5'.format(SAT))[0]
except:
hdfeos5_file = None
if ut.run_or_skip(
out_file=hdfeos5_file,
in_file=[ts_file, tcoh_file, mask_file,
geom_file]) == 'run':
mintpy.save_hdfeos5.main(scp_args.split())
else:
print('save time-series to HDF-EOS5 format is OFF.')
return
def main(iargs=None):
inps = cmd_line_parse(iargs)
# --update option
if inps.update_mode and run_or_skip(inps) == 'skip':
return inps.outfile
out_file = ut.run_deramp(inps.file,
ramp_type=inps.surface_type,
mask_file=inps.mask_file,
out_file=inps.outfile,
datasetName=inps.dset)
# config parameter
print('add/update the following configuration metadata to file:\n{}'.format(configKeys))
atr_new = {}
atr_new['mintpy.deramp'] = inps.surface_type
atr_new['mintpy.deramp.maskFile'] = inps.mask_file
ut.add_attribute(out_file, atr_new)
return
def main(iargs=None):
inps = cmd_line_parse(iargs)
# --update option
if inps.update_mode and run_or_skip(inps) == 'skip':
return inps.outfile
out_file = ut.run_deramp(inps.file,
ramp_type=inps.surface_type,
mask_file=inps.mask_file,
out_file=inps.outfile,
datasetName=inps.dset)
# config parameter
print(
'add/update the following configuration metadata to file:\n{}'.format(
configKeys))
atr_new = {}
atr_new['mintpy.deramp'] = inps.surface_type
atr_new['mintpy.deramp.maskFile'] = inps.mask_file
ut.add_attribute(out_file, atr_new)
return
def run_save2hdfeos5(self, step_name):
"""Save displacement time-series and its aux data in geo coordinate into HDF-EOS5 format"""
if self.template['mintpy.save.hdfEos5'] is True:
# input
ts_file = self.get_timeseries_filename(self.template)[step_name]['input']
# Add attributes from custom template to timeseries file
if self.customTemplate is not None:
ut.add_attribute(ts_file, self.customTemplate)
tcoh_file = 'temporalCoherence.h5'
mask_file = 'geo_maskTempCoh.h5'
geom_file = ut.check_loaded_dataset(self.workDir, print_msg=False)[2]
if 'geo' in ts_file:
tcoh_file = './geo/geo_temporalCoherence.h5'
mask_file = './geo/geo_maskTempCoh.h5'
geom_file = './geo/geo_{}'.format(os.path.basename(geom_file))
# cmd
print('--------------------------------------------')
scp_args = '{f} -c {c} -m {m} -g {g} -t {t}'.format(f=ts_file,
c=tcoh_file,
m=mask_file,
g=geom_file,
t=self.templateFile)
print('save_hdfeos5.py', scp_args)
# output (check existing file)
atr = readfile.read_attribute(ts_file)
SAT = sensor.get_unavco_mission_name(atr)
try:
hdfeos5_file = get_file_list('{}_*.he5'.format(SAT))[0]
except:
hdfeos5_file = None
if ut.run_or_skip(out_file=hdfeos5_file, in_file=[ts_file, tcoh_file, mask_file, geom_file]) == 'run':
mintpy.save_hdfeos5.main(scp_args.split())
else:
print('save time-series to HDF-EOS5 format is OFF.')
return
def prepare_metadata4giant(fname, meta_files=None):
"""Extract metadata from xml files for GIAnT time-series file."""
# check xml files
if not meta_files:
meta_files = auto_xml_file4giant(fname)
if not meta_files:
raise FileNotFoundError("no xml file found.")
# extract metadata from xml files
rsc_files = [i for i in meta_files if i.endswith('.rsc')]
xml_files = [i for i in meta_files if i.endswith('.xml')]
xml_dict = {}
for rsc_file in rsc_files:
print('reading {}'.format(rsc_file))
rsc_dict = readfile.read_roipac_rsc(rsc_file)
for key in ['length', 'LENGTH', 'FILE_LENGTH', 'width', 'WIDTH']:
try:
rsc_dict.pop(key)
except:
pass
xml_dict.update(rsc_dict)
for xml_file in xml_files:
print('reading {}'.format(xml_file))
xml_dict.update(read_giant_xml(xml_file))
if not xml_dict:
raise ValueError('No metadata found in file: ' + xml_file)
# standardize metadata names
xml_dict = readfile.standardize_metadata(xml_dict)
# project name
sensor_name, project_name = sensor.project_name2sensor_name(
os.path.abspath(fname))
if sensor_name:
xml_dict['PLATFORM'] = sensor_name
if project_name:
xml_dict['PROJECT_NAME'] = project_name
if sensor_name in project_name:
tmp = project_name.split(sensor_name)[1][0]
if tmp == 'A':
xml_dict['ORBIT_DIRECTION'] = 'ASCENDING'
else:
xml_dict['ORBIT_DIRECTION'] = 'DESCENDING'
# update GIAnT HDF5 file
fname = ut.add_attribute(fname, xml_dict, print_msg=True)
return fname
def prepare_metadata4giant(fname, meta_files=None):
"""Extract metadata from xml files for GIAnT time-series file."""
# check xml files
if not meta_files:
meta_files = auto_xml_file4giant(fname)
if not meta_files:
raise FileNotFoundError("no xml file found.")
# extract metadata from xml files
rsc_files = [i for i in meta_files if i.endswith('.rsc')]
xml_files = [i for i in meta_files if i.endswith('.xml')]
xml_dict = {}
for rsc_file in rsc_files:
print('reading {}'.format(rsc_file))
rsc_dict = readfile.read_roipac_rsc(rsc_file)
for key in ['length', 'LENGTH', 'FILE_LENGTH', 'width', 'WIDTH']:
try:
rsc_dict.pop(key)
except:
pass
xml_dict.update(rsc_dict)
for xml_file in xml_files:
print('reading {}'.format(xml_file))
xml_dict.update(read_giant_xml(xml_file))
if not xml_dict:
raise ValueError('No metadata found in file: '+xml_file)
# standardize metadata names
xml_dict = readfile.standardize_metadata(xml_dict)
# project name
sensor_name, project_name = sensor.project_name2sensor_name(os.path.abspath(fname))
if sensor_name:
xml_dict['PLATFORM'] = sensor_name
if project_name:
xml_dict['PROJECT_NAME'] = project_name
if sensor_name in project_name:
tmp = project_name.split(sensor_name)[1][0]
if tmp == 'A':
xml_dict['ORBIT_DIRECTION'] = 'ASCENDING'
else:
xml_dict['ORBIT_DIRECTION'] = 'DESCENDING'
# update GIAnT HDF5 file
fname = ut.add_attribute(fname, xml_dict, print_msg=True)
return fname
def main(iargs=None):
inps = cmd_line_parse(iargs)
dateStr = datetime.datetime.strftime(datetime.datetime.now(),
'%Y%m%d:%H%M%S')
if not iargs is None:
msg = os.path.basename(__file__) + ' ' + ' '.join(iargs[:])
string = dateStr + " * " + msg
print(string)
else:
msg = os.path.basename(__file__) + ' ' + ' '.join(sys.argv[1::])
string = dateStr + " * " + msg
print(string)
work_dir = os.path.dirname(inps.outdir)
#os.chdir(work_dir)
# read input options
inpsDict = read_inps2dict(inps)
prepare_metadata(inpsDict)
inpsDict = read_subset_box(inpsDict)
extraDict = get_extra_metadata(inpsDict)
if not 'PLATFORM' in extraDict:
slcStack = os.path.join(os.path.dirname(work_dir),
'inputs/slcStack.h5')
atr = readfile.read_attribute(slcStack)
if 'PLATFORM' in atr:
extraDict['PLATFORM'] = atr['PLATFORM']
# initiate objects
inpsDict['ds_name2key'] = datasetName2templateKey
stackObj = mld.read_inps_dict2ifgram_stack_dict_object(inpsDict)
# prepare wirte
updateMode, comp, box, boxGeo = print_write_setting(inpsDict)
box = None
boxGeo = None
if stackObj and not os.path.isdir(inps.outdir):
os.makedirs(inps.outdir)
print('create directory: {}'.format(inps.outdir))
# write
if stackObj and mld.update_object(inps.outfile[0],
stackObj,
box,
updateMode=updateMode,
xstep=inpsDict['xstep'],
ystep=inpsDict['ystep']):
print('-' * 50)
stackObj.write2hdf5(outputFile=inps.outfile[0],
access_mode='w',
box=box,
compression=comp,
extra_metadata=extraDict)
geo_files = ['geometryRadar.h5', 'geometryGeo.h5']
copy_file = False
for geometry_file_2 in geo_files:
geometry_file = os.path.join(os.path.dirname(work_dir), 'inputs',
geometry_file_2)
if os.path.exists(geometry_file):
copy_file = True
break
if copy_file:
if not os.path.exists(
os.path.join(work_dir, 'inputs/{}'.format(geometry_file_2))):
shutil.copyfile(
geometry_file,
os.path.join(work_dir, 'inputs/{}'.format(geometry_file_2)))
else:
geomRadarObj, geomGeoObj = mld.read_inps_dict2geometry_dict_object(
inpsDict)
if geomRadarObj and mld.update_object(inps.outfile[1],
geomRadarObj,
box,
updateMode=updateMode,
xstep=inpsDict['xstep'],
ystep=inpsDict['ystep']):
print('-' * 50)
geomRadarObj.write2hdf5(outputFile=inps.outfile[1],
access_mode='w',
box=box,
xstep=inpsDict['xstep'],
ystep=inpsDict['ystep'],
compression='lzf',
extra_metadata=extraDict)
if geomGeoObj and mld.update_object(inps.outfile[2],
geomGeoObj,
boxGeo,
updateMode=updateMode,
xstep=inpsDict['xstep'],
ystep=inpsDict['ystep']):
print('-' * 50)
geomGeoObj.write2hdf5(outputFile=inps.outfile[2],
access_mode='w',
box=boxGeo,
xstep=inpsDict['xstep'],
ystep=inpsDict['ystep'],
compression='lzf')
# check loading result
load_complete, stack_file, geom_file = ut.check_loaded_dataset(
work_dir=work_dir, print_msg=True)[0:3]
# add custom metadata (optional)
customTemplate = inps.template_file[0]
if customTemplate:
print('updating {}, {} metadata based on custom template file: {}'.
format(os.path.basename(stack_file), os.path.basename(geom_file),
os.path.basename(customTemplate)))
# use ut.add_attribute() instead of add_attribute.py because of
# better control of special metadata, such as SUBSET_X/YMIN
ut.add_attribute(stack_file, inpsDict)
ut.add_attribute(geom_file, inpsDict)
ut.add_attribute(stack_file, extraDict)
# if not load_complete, plot and raise exception
if not load_complete:
# go back to original directory
print('Go back to directory:', work_dir)
os.chdir(work_dir)
# raise error
msg = 'step load_ifgram: NOT all required dataset found, exit.'
raise SystemExit(msg)
return inps.outfile
def main(iargs=None):
"""
Overwrite filtered SLC images in Isce merged/SLC directory.
"""
Parser = MinoPyParser(iargs, script='generate_temporal_coherence')
inps = Parser.parse()
dateStr = datetime.datetime.strftime(datetime.datetime.now(),
'%Y%m%d:%H%M%S')
if not iargs is None:
msg = os.path.basename(__file__) + ' ' + ' '.join(iargs[:])
string = dateStr + " * " + msg
print(string)
else:
msg = os.path.basename(__file__) + ' ' + ' '.join(sys.argv[1::])
string = dateStr + " * " + msg
print(string)
start_time = time.time()
os.chdir(inps.work_dir)
minopy_dir = os.path.dirname(inps.work_dir)
minopy_template_file = os.path.join(minopy_dir, 'minopyApp.cfg')
inps.ifgramStackFile = os.path.join(inps.work_dir, 'inputs/ifgramStack.h5')
template = readfile.read_template(minopy_template_file)
if template['minopy.timeseries.tempCohType'] == 'auto':
template['minopy.timeseries.tempCohType'] = 'full'
atr = {}
atr['minopy.timeseries.tempCohType'] = template[
'minopy.timeseries.tempCohType']
ut.add_attribute(inps.ifgramStackFile, atr)
# check if input observation dataset exists.
stack_obj = ifgramStack(inps.ifgramStackFile)
stack_obj.open(print_msg=False)
metadata = stack_obj.get_metadata()
length, width = stack_obj.length, stack_obj.width
inps.invQualityFile = 'temporalCoherence.h5'
mintpy_mask_file = os.path.join(inps.work_dir, 'maskTempCoh.h5')
quality_name = os.path.join(
minopy_dir, 'inverted/tempCoh_{}'.format(
template['minopy.timeseries.tempCohType']))
quality = np.memmap(quality_name,
mode='r',
dtype='float32',
shape=(length, width))
# inps.waterMaskFile = os.path.join(minopy_dir, 'waterMask.h5')
inps.waterMaskFile = None
water_mask = np.ones(quality.shape, dtype=np.int8)
if template['minopy.timeseries.waterMask'] != 'auto':
inps.waterMaskFile = template['minopy.timeseries.waterMask']
if os.path.exists(inps.waterMaskFile):
with h5py.File(inps.waterMaskFile, 'r') as f2:
if 'waterMask' in f2:
water_mask = f2['waterMask'][:, :]
else:
water_mask = f2['mask'][:, :]
if inps.shadow_mask:
if os.path.exists(os.path.join(minopy_dir, 'shadow_mask.h5')):
with h5py.File(os.path.join(minopy_dir, 'shadow_mask.h5'),
'r') as f2:
shadow_mask = f2['mask'][:, :]
water_mask = water_mask * shadow_mask
inv_quality = np.zeros((quality.shape[0], quality.shape[1]))
inv_quality_name = 'temporalCoherence'
inv_quality[:, :] = quality[:, :]
inv_quality[inv_quality <= 0] = np.nan
inv_quality[water_mask < 0.5] = np.nan
if os.path.exists(mintpy_mask_file):
mintpy_mask = readfile.read(mintpy_mask_file, datasetName='mask')[0]
inv_quality[mintpy_mask == 0] = np.nan
if not os.path.exists(inps.invQualityFile):
metadata['UNIT'] = '1'
metadata['FILE_TYPE'] = inv_quality_name
if 'REF_DATE' in metadata:
metadata.pop('REF_DATE')
ds_name_dict = {metadata['FILE_TYPE']: [np.float32, (length, width)]}
writefile.layout_hdf5(inps.invQualityFile,
ds_name_dict,
metadata=metadata)
# write the block to disk
# with 3D block in [z0, z1, y0, y1, x0, x1]
# and 2D block in [y0, y1, x0, x1]
block = [0, length, 0, width]
writefile.write_hdf5_block(inps.invQualityFile,
data=inv_quality,
datasetName=inv_quality_name,
block=block)
get_phase_linking_coherence_mask(metadata, inps.work_dir)
m, s = divmod(time.time() - start_time, 60)
print('time used: {:02.0f} mins {:02.1f} secs.\n'.format(m, s))
return
def run_load_data(self, step_name):
"""Load InSAR stacks into HDF5 files in ./inputs folder.
It 1) copy auxiliary files into work directory (for Unvi of Miami only)
2) load all interferograms stack files into mintpy/inputs directory.
3) check loading result
4) add custom metadata (optional, for HDF-EOS5 format only)
"""
# 1) copy aux files (optional)
self._copy_aux_file()
# 2) loading data
stack_processor = self.template['mintpy.load.processor'].lower()
if stack_processor == 'aria':
# use subprocess instead of main() here to avoid import gdal3
# which is required in prep_aria.py
cmd = 'prep_aria.py --template {} --update '.format(self.templateFile)
print(cmd)
subprocess.Popen(cmd, shell=True).wait()
else:
# compose command line
scp_args = '--template {}'.format(self.templateFile)
if self.customTemplateFile:
scp_args += ' {}'.format(self.customTemplateFile)
if self.projectName:
scp_args += ' --project {}'.format(self.projectName)
# run command line
print("load_data.py", scp_args)
mintpy.load_data.main(scp_args.split())
# come back to working directory
os.chdir(self.workDir)
# 3) check loading result
load_complete, stack_file, geom_file = ut.check_loaded_dataset(self.workDir, print_msg=True)[0:3]
# 4) add custom metadata (optional)
if self.customTemplateFile:
print('updating {}, {} metadata based on custom template file: {}'.format(
os.path.basename(stack_file),
os.path.basename(geom_file),
os.path.basename(self.customTemplateFile)))
# use ut.add_attribute() instead of add_attribute.py because of
# better control of special metadata, such as SUBSET_X/YMIN
ut.add_attribute(stack_file, self.customTemplate)
ut.add_attribute(geom_file, self.customTemplate)
# 5) if not load_complete, plot and raise exception
if not load_complete:
# plot result if error occured
self.plot_result(print_aux=False, plot=plot)
# go back to original directory
print('Go back to directory:', self.cwd)
os.chdir(self.cwd)
# raise error
msg = 'step {}: NOT all required dataset found, exit.'.format(step_name)
raise RuntimeError(msg)
return
def reference_file(inps):
"""Seed input file with option from input namespace
Return output file name if succeed; otherwise, return None
"""
if not inps:
inps = cmd_line_parse([''])
atr = readfile.read_attribute(inps.file)
# update_mode
if (not inps.force and inps.ref_y is not None
and inps.ref_y == int(atr.get('REF_Y', -999))
and inps.ref_x is not None
and inps.ref_x == int(atr.get('REF_X', -999))):
print(
'SAME reference pixel is already selected/saved in file, skip updating.'
)
return inps.file
# Check 1 - stack and its non-nan mask pixel coverage
stack = ut.temporal_average(inps.file,
datasetName='unwrapPhase',
updateMode=True,
outFile=False)[0]
mask = np.multiply(~np.isnan(stack), stack != 0.)
if np.nansum(mask) == 0.0:
raise ValueError(
'no pixel found with valid phase value in all datasets.')
# Check 2 - input ref_y/x: location and validity
if inps.ref_y is not None and inps.ref_x is not None:
if mask[inps.ref_y, inps.ref_x] == 0.:
raise ValueError(
'reference y/x have nan value in some dataset. Please re-select.'
)
else:
# Find reference y/x
if inps.method == 'maxCoherence':
inps.ref_y, inps.ref_x = select_max_coherence_yx(
coh_file=inps.coherenceFile,
mask=mask,
min_coh=inps.minCoherence)
elif inps.method == 'random':
inps.ref_y, inps.ref_x = random_select_reference_yx(mask)
elif inps.method == 'manual':
inps = manual_select_reference_yx(stack, inps, mask)
# Check ref_y/x from auto method
if inps.ref_y is None or inps.ref_x is None:
raise ValueError('ERROR: no reference y/x found.')
# Seeding file with reference y/x
atrNew = reference_point_attribute(atr, y=inps.ref_y, x=inps.ref_x)
if not inps.write_data:
print('Add/update ref_x/y attribute to file: ' + inps.file)
print(atrNew)
inps.outfile = ut.add_attribute(inps.file, atrNew)
else:
if not inps.outfile:
inps.outfile = inps.file
k = atr['FILE_TYPE']
fext = os.path.splitext(inps.file)[1]
if fext == '.h5':
if inps.outfile == inps.file:
print('updating data value without re-writing to a new file')
if k == 'ifgramStack':
with h5py.File(inps.file, 'r+') as f:
ds = f['unwrapPhase']
for i in range(ds.shape[0]):
ds[i, :, :] -= ds[i, inps.ref_y, inps.ref_x]
print('update metadata')
f.attrs.update(atrNew)
else:
with h5py.File(inps.file, 'r+') as f:
ds = f[k]
if len(ds.shape) == 3:
# 3D matrix
for i in range(ds.shape[0]):
ds[i, :, :] -= ds[i, inps.ref_y, inps.ref_x]
else:
# 2D matrix
ds[:] -= ds[inps.ref_y, inps.ref_x]
print('update metadata')
f.attrs.update(atrNew)
else:
## write to a new file
print('writing the referenced data into file: {}'.format(
inps.outfile))
# 1. read and update data value
data, atr = readfile.read(inps.file)
if len(data.shape) == 3:
# 3D matrix
for i in range(data.shape[0]):
data[i, :, :] -= data[i, inps.ref_y, inps.ref_x]
else:
# 2D matrix
data -= data[inps.ref_y, inps.ref_x]
# 2. update metadata
atr.update(atrNew)
# 3. write to file
writefile.write(data,
inps.outfile,
metadata=atr,
ref_file=inps.file)
else:
# for binary file, over-write directly
data = readfile.read(inps.file)[0]
data -= data[inps.ref_y, inps.ref_x]
atr.update(atrNew)
writefile.write(data, out_file=inps.outfile, metadata=atr)
ut.touch([inps.coherenceFile, inps.maskFile])
return inps.outfile
def reference_file(inps):
"""Seed input file with option from input namespace
Return output file name if succeed; otherwise, return None
"""
if not inps:
inps = cmd_line_parse([''])
atr = readfile.read_attribute(inps.file)
if (inps.ref_y and inps.ref_x and 'REF_Y' in atr.keys()
and inps.ref_y == int(atr['REF_Y'])
and inps.ref_x == int(atr['REF_X']) and not inps.force):
print(
'Same reference pixel is already selected/saved in file, skip updating.'
)
return inps.file
# Get stack and mask
stack = ut.temporal_average(inps.file,
datasetName='unwrapPhase',
updateMode=True,
outFile=False)[0]
mask = np.multiply(~np.isnan(stack), stack != 0.)
if np.nansum(mask) == 0.0:
raise ValueError(
'no pixel found with valid phase value in all datasets.')
if inps.ref_y and inps.ref_x and mask[inps.ref_y, inps.ref_x] == 0.:
raise ValueError(
'reference y/x have nan value in some dataset. Please re-select.')
# Find reference y/x
if not inps.ref_y or not inps.ref_x:
if inps.method == 'maxCoherence':
inps.ref_y, inps.ref_x = select_max_coherence_yx(
coh_file=inps.coherenceFile,
mask=mask,
min_coh=inps.minCoherence)
elif inps.method == 'random':
inps.ref_y, inps.ref_x = random_select_reference_yx(mask)
elif inps.method == 'manual':
inps = manual_select_reference_yx(stack, inps, mask)
if not inps.ref_y or not inps.ref_x:
raise ValueError('ERROR: no reference y/x found.')
# Seeding file with reference y/x
atrNew = reference_point_attribute(atr, y=inps.ref_y, x=inps.ref_x)
if not inps.write_data:
print('Add/update ref_x/y attribute to file: ' + inps.file)
print(atrNew)
inps.outfile = ut.add_attribute(inps.file, atrNew)
else:
if not inps.outfile:
inps.outfile = '{}_seeded{}'.format(
os.path.splitext(inps.file)[0],
os.path.splitext(inps.file)[1])
k = atr['FILE_TYPE']
# For ifgramStack file, update data value directly, do not write to new file
if k == 'ifgramStack':
f = h5py.File(inps.file, 'r+')
ds = f[k].get('unwrapPhase')
for i in range(ds.shape[0]):
ds[i, :, :] -= ds[i, inps.ref_y, inps.ref_x]
f[k].attrs.update(atrNew)
f.close()
inps.outfile = inps.file
elif k == 'timeseries':
data = timeseries(inps.file).read()
for i in range(data.shape[0]):
data[i, :, :] -= data[i, inps.ref_y, inps.ref_x]
obj = timeseries(inps.outfile)
atr.update(atrNew)
obj.write2hdf5(data=data, metadata=atr, refFile=inps.file)
obj.close()
else:
print('writing >>> ' + inps.outfile)
data = readfile.read(inps.file)[0]
data -= data[inps.ref_y, inps.ref_x]
atr.update(atrNew)
writefile.write(data, out_file=inps.outfile, metadata=atr)
ut.touch([inps.coherenceFile, inps.maskFile])
return inps.outfile
def reference_file(inps):
"""Seed input file with option from input namespace
Return output file name if succeed; otherwise, return None
"""
if not inps:
inps = cmd_line_parse([''])
atr = readfile.read_attribute(inps.file)
if (inps.ref_y and inps.ref_x and 'REF_Y' in atr.keys()
and inps.ref_y == int(atr['REF_Y']) and inps.ref_x == int(atr['REF_X'])
and not inps.force):
print('Same reference pixel is already selected/saved in file, skip updating.')
return inps.file
# Get stack and mask
stack = ut.temporal_average(inps.file, datasetName='unwrapPhase', updateMode=True, outFile=False)[0]
mask = np.multiply(~np.isnan(stack), stack != 0.)
if np.nansum(mask) == 0.0:
raise ValueError('no pixel found with valid phase value in all datasets.')
if inps.ref_y and inps.ref_x and mask[inps.ref_y, inps.ref_x] == 0.:
raise ValueError('reference y/x have nan value in some dataset. Please re-select.')
# Find reference y/x
if not inps.ref_y or not inps.ref_x:
if inps.method == 'maxCoherence':
inps.ref_y, inps.ref_x = select_max_coherence_yx(coh_file=inps.coherenceFile,
mask=mask,
min_coh=inps.minCoherence)
elif inps.method == 'random':
inps.ref_y, inps.ref_x = random_select_reference_yx(mask)
elif inps.method == 'manual':
inps = manual_select_reference_yx(stack, inps, mask)
if not inps.ref_y or not inps.ref_x:
raise ValueError('ERROR: no reference y/x found.')
# Seeding file with reference y/x
atrNew = reference_point_attribute(atr, y=inps.ref_y, x=inps.ref_x)
if not inps.write_data:
print('Add/update ref_x/y attribute to file: '+inps.file)
print(atrNew)
inps.outfile = ut.add_attribute(inps.file, atrNew)
else:
if not inps.outfile:
inps.outfile = '{}_seeded{}'.format(os.path.splitext(inps.file)[0],
os.path.splitext(inps.file)[1])
k = atr['FILE_TYPE']
# For ifgramStack file, update data value directly, do not write to new file
if k == 'ifgramStack':
f = h5py.File(inps.file, 'r+')
ds = f[k].get('unwrapPhase')
for i in range(ds.shape[0]):
ds[i, :, :] -= ds[i, inps.ref_y, inps.ref_x]
f[k].attrs.update(atrNew)
f.close()
inps.outfile = inps.file
elif k == 'timeseries':
data = timeseries(inps.file).read()
for i in range(data.shape[0]):
data[i, :, :] -= data[i, inps.ref_y, inps.ref_x]
obj = timeseries(inps.outfile)
atr.update(atrNew)
obj.write2hdf5(data=data, metadata=atr, refFile=inps.file)
obj.close()
else:
print('writing >>> '+inps.outfile)
data = readfile.read(inps.file)[0]
data -= data[inps.ref_y, inps.ref_x]
atr.update(atrNew)
writefile.write(data, out_file=inps.outfile, metadata=atr)
ut.touch([inps.coherenceFile, inps.maskFile])
return inps.outfile
def main(iargs=None):
"""
Overwrite filtered SLC images in Isce merged/SLC directory.
"""
Parser = MinoPyParser(iargs, script='invert_network')
inps = Parser.parse()
dateStr = datetime.datetime.strftime(datetime.datetime.now(),
'%Y%m%d:%H%M%S')
if not iargs is None:
msg = os.path.basename(__file__) + ' ' + ' '.join(iargs[:])
string = dateStr + " * " + msg
print(string)
else:
msg = os.path.basename(__file__) + ' ' + ' '.join(sys.argv[1::])
string = dateStr + " * " + msg
print(string)
start_time = time.time()
os.chdir(inps.work_dir)
minopy_dir = os.path.dirname(inps.work_dir)
if inps.template_file is None:
inps.template_file = os.path.join(minopy_dir, 'smallbaselineApp.cfg')
minopy_template_file = os.path.join(minopy_dir, 'minopyApp.cfg')
inps.ifgramStackFile = os.path.join(inps.work_dir, 'inputs/ifgramStack.h5')
template = readfile.read_template(minopy_template_file)
if template['minopy.timeseries.tempCohType'] == 'auto':
template['minopy.timeseries.tempCohType'] = 'full'
atr = {}
atr['minopy.timeseries.tempCohType'] = template[
'minopy.timeseries.tempCohType']
ut.add_attribute(inps.ifgramStackFile, atr)
# 1) invert ifgramStack for time-series
stack_file = ut.check_loaded_dataset(inps.work_dir, print_msg=False)[1]
#wrapped_phase_series = os.path.join(minopy_dir, 'inverted/phase_series.h5')
iargs = [
stack_file, '-t', inps.template_file, '--update', '--norm',
inps.residualNorm, '--tcoh', inps.temp_coh, '--mask-threshold',
str(inps.maskThreshold), '--smooth_factor', inps.L1_alpha
] #, '--calc-cov']
if not inps.minNormVelocity:
iargs += ['--min-norm-phase']
print('\nifgram_inversion_L1L2.py', ' '.join(iargs))
ifgram_inversion_L1L2.main(iargs)
# 1) Replace temporal coherence with the one obtained from full stack inversion
iargs = ['-d', inps.work_dir]
if inps.shadow_mask:
iargs = ['-d', inps.work_dir, '--shadow_mask']
print('\ngenerate_temporal_coherence.py', ' '.join(iargs))
generate_temporal_coherence.main(iargs)
#m, s = divmod(time.time() - start_time, 60)
#print('time used: {:02.0f} mins {:02.1f} secs.\n'.format(m, s))
return
def main(iargs=None):
inps = cmd_line_parse(iargs)
key = 'geolocation_corrected'
with h5py.File(inps.geometry_file, 'r') as f:
keys = f.attrs.keys()
latitude = f['latitude'][:, :]
longitude = f['longitude'][:, :]
atr = readfile.read(inps.geometry_file, datasetName='azimuthAngle')[1]
if not key in keys or atr[key] == 'no':
status = 'run'
print('Run geolocation correction ...')
else:
status = 'skip'
print(
'Geolocation is already done, you may reverse it using --reverse. skip ...'
)
if inps.reverse:
if key in keys and atr[key] == 'yes':
status = 'run'
print('Run reversing geolocation correction ...')
else:
status = 'skip'
print('The file is not corrected for geolocation. skip ...')
if status == 'run':
az_angle = np.deg2rad(np.float(atr['HEADING']))
inc_angle = np.deg2rad(
readfile.read(inps.geometry_file, datasetName='incidenceAngle')[0])
dem_error = readfile.read(inps.dem_error_file, datasetName='dem')[0]
rad_latitude = np.deg2rad(latitude)
one_degree_latitude = 111132.92 - 559.82 * np.cos(2*rad_latitude) + \
1.175 * np.cos(4 * rad_latitude) - 0.0023 * np.cos(6 * rad_latitude)
one_degree_longitude = 111412.84 * np.cos(rad_latitude) - \
93.5 * np.cos(3 * rad_latitude) + 0.118 * np.cos(5 * rad_latitude)
dx = np.divide(dem_error * (1 / np.tan(inc_angle)) * np.cos(az_angle),
one_degree_longitude) # converted to degree
dy = np.divide(dem_error * (1 / np.tan(inc_angle)) * np.sin(az_angle),
one_degree_latitude) # converted to degree
if inps.reverse:
sign = np.sign(latitude)
latitude -= sign * dy
sign = np.sign(longitude)
if atr['ORBIT_DIRECTION'] == 'Ascending':
longitude += sign * dx
else:
longitude -= sign * dx
atr[key] = 'no'
block = [0, latitude.shape[0], 0, latitude.shape[1]]
writefile.write_hdf5_block(inps.geometry_file,
data=latitude,
datasetName='latitude',
block=block)
writefile.write_hdf5_block(inps.geometry_file,
data=longitude,
datasetName='longitude',
block=block)
ut.add_attribute(inps.geometry_file, atr_new=atr)
else:
sign = np.sign(latitude)
latitude += sign * dy
sign = np.sign(longitude)
if atr['ORBIT_DIRECTION'] == 'Ascending':
longitude -= sign * dx
else:
longitude += sign * dx
atr[key] = 'yes'
block = [0, latitude.shape[0], 0, latitude.shape[1]]
writefile.write_hdf5_block(inps.geometry_file,
data=latitude,
datasetName='latitude',
block=block)
writefile.write_hdf5_block(inps.geometry_file,
data=longitude,
datasetName='longitude',
block=block)
ut.add_attribute(inps.geometry_file, atr_new=atr)
f.close()
return
