def startup(self):
"""The starting point of the workflow. It runs everytime.
It 1) grab project name if given
2) grab and go to work directory
3) get and read template(s) options
"""
#1. Get projectName
self.projectName = None
if self.customTemplateFile:
self.projectName = os.path.splitext(
os.path.basename(self.customTemplateFile))[0]
print('Project name:', self.projectName)
#2. Go to the work directory
#2.1 Get workDir
if not self.workDir:
if autoPath and 'SCRATCHDIR' in os.environ and self.projectName:
self.workDir = os.path.join(os.getenv('SCRATCHDIR'),
self.projectName, 'PYSAR')
else:
self.workDir = os.getcwd()
self.workDir = os.path.abspath(self.workDir)
#2.2 Go to workDir
if not os.path.isdir(self.workDir):
os.makedirs(self.workDir)
print('create directory:', self.workDir)
os.chdir(self.workDir)
print("Go to work directory:", self.workDir)
#3. Read templates
#3.1 Get default template file
lfile = os.path.join(os.path.dirname(__file__),
'defaults/pysarApp_template.txt') #latest version
cfile = os.path.join(self.workDir,
'pysarApp_template.txt') #current version
if not os.path.isfile(cfile):
print('copy default template file {} to work directory'.format(
lfile))
shutil.copy2(lfile, self.workDir)
else:
#cfile is obsolete if any key is missing
ldict = readfile.read_template(lfile)
cdict = readfile.read_template(cfile)
if any([key not in cdict.keys() for key in ldict.keys()]):
print(
'obsolete default template detected, update to the latest version.'
)
shutil.copy2(lfile, self.workDir)
#keep the existing option value from obsolete template file
ut.update_template_file(cfile, cdict)
self.templateFile = cfile
# 3.2 read (custom) template files into dicts
self._read_template()
return
def update_template_file(template_file, extra_dict):
"""Update option value in template_file with value from input extra_dict"""
# Compare and skip updating template_file if no new option value found.
update = False
orig_dict = readfile.read_template(template_file)
for key, value in orig_dict.items():
if key in extra_dict.keys() and extra_dict[key] != value:
update = True
if not update:
print('No new option value found, skip updating '+template_file)
return template_file
# Update template_file with new value from extra_dict
tmp_file = template_file+'.tmp'
f_tmp = open(tmp_file, 'w')
for line in open(template_file, 'r'):
c = [i.strip() for i in line.strip().split('=', 1)]
if not line.startswith(('%', '#')) and len(c) > 1:
key = c[0]
value = str.replace(c[1], '\n', '').split("#")[0].strip()
if key in extra_dict.keys() and extra_dict[key] != value:
line = line.replace(value, extra_dict[key], 1)
print(' {}: {} --> {}'.format(key, value, extra_dict[key]))
f_tmp.write(line)
f_tmp.close()
# Overwrite exsting original template file
mvCmd = 'mv {} {}'.format(tmp_file, template_file)
os.system(mvCmd)
return template_file
def read_template2inps(templateFile, inps=None):
"""Update inps with pysar.residualRms.* option from templateFile"""
if not inps:
inps = cmd_line_parse()
template = readfile.read_template(templateFile)
prefix = 'pysar.residualRms.'
key = prefix + 'maskFile'
if key in template.keys():
value = template[key]
if value == 'auto':
inps.mask_file = 'maskTempCoh.h5'
elif value == 'no':
inps.mask_file = None
else:
inps.mask_file = value
key = prefix + 'ramp'
if key in template.keys():
value = template[key]
if value == 'auto':
inps.ramp_type = 'quadratic'
else:
inps.ramp_type = value
key = prefix + 'threshold'
if key in template.keys():
value = template[key]
if value == 'auto':
inps.min_rms = 0.02
else:
inps.min_rms = float(value)
return inps
def read_template_file2inps(template_file, inps=None):
"""Read seed/reference info from template file and update input namespace"""
if not inps:
inps = cmd_line_parse([''])
inps_dict = vars(inps)
template = readfile.read_template(template_file)
template = ut.check_template_auto_value(template)
prefix = 'pysar.reference.'
key_list = [i for i in list(inps_dict) if prefix + i in template.keys()]
for key in key_list:
value = template[prefix + key]
if value:
if key in ['coherenceFile', 'maskFile']:
inps_dict[key] = value
elif key == 'minCoherence':
inps_dict[key] = float(value)
key = prefix + 'yx'
if key in template.keys():
value = template[key]
if value:
inps.ref_y, inps.ref_x = [int(i) for i in value.split(',')]
key = prefix + 'lalo'
if key in template.keys():
value = template[key]
if value:
inps.ref_lat, inps.ref_lon = [float(i) for i in value.split(',')]
return inps
def main(argv):
parser = build_parser()
parseArgs = parser.parse_args()
username = parseArgs.user
password = parseArgs.password
host = parseArgs.host
db = parseArgs.db
working_dir = parseArgs.folder
# make sure we have a final / so the below code doesn't break
if working_dir[-1] != "/":
working_dir += "/"
unavco_name = parseArgs.unavco_name
attributes_file = working_dir + "add_Attribute.txt"
attributes = readfile.read_template(attributes_file)
dbController = InsarDatabaseController(username, password, host, db)
dbController.connect()
for key in attributes:
print("Setting attribute " + key + " to " + attributes[key])
if key == "plotAttributes":
dbController.add_plot_attribute(unavco_name, key, attributes[key])
else:
dbController.add_attribute(unavco_name, key, attributes[key])
dbController.index_table_on("extra_attributes", "area_id", "area_id_idx")
dbController.close()
def read_template2inps(template_file, inps):
"""Read input template options into Namespace inps"""
print('read input option from template file: ' + template_file)
if not inps:
inps = cmd_line_parse()
inps_dict = vars(inps)
template = readfile.read_template(template_file)
template = ut.check_template_auto_value(template)
prefix = 'pysar.geocode.'
key_list = [i for i in list(inps_dict.keys()) if prefix + i in template.keys()]
for key in key_list:
value = template[prefix + key]
if value:
if key == 'SNWE':
inps_dict[key] = tuple([float(i) for i in value.split(',')])
elif key in ['latStep', 'lonStep']:
inps_dict[key] = float(value)
elif key in ['interpMethod']:
inps_dict[key] = value
elif key == 'fillValue':
if 'nan' in value.lower():
inps_dict[key] = np.nan
else:
inps_dict[key] = float(value)
inps.laloStep = [inps.latStep, inps.lonStep]
if None in inps.laloStep:
inps.laloStep = None
return inps
def check_obsolete_default_template(inps):
"""Update pysarApp_template.txt file if it's obsolete, a.k.a. lack new option names"""
template_file = os.path.join(inps.workDir, 'pysarApp_template.txt')
obsolete_template = False
current_dict = readfile.read_template(template_file)
latest_dict = readfile.read_template(inps.autoTemplateFile)
for key in latest_dict.keys():
if key not in current_dict.keys():
obsolete_template = True
if obsolete_template:
print('obsolete default template detected, update to the latest template options.')
shutil.copy2(inps.autoTemplateFile, inps.workDir)
template_file = ut.update_template_file(template_file, current_dict)
else:
print('latest template file detected:', template_file)
return template_file
def check_obsolete_default_template(template_file='./pysarApp_template.txt'):
"""Update pysarApp_template.txt file if it's obsolete, a.k.a. lack new option names"""
obsolete_template = False
current_dict = readfile.read_template(template_file)
latest_dict = readfile.read_template(TEMPLATE)
for key in latest_dict.keys():
if key not in current_dict.keys():
obsolete_template = True
if obsolete_template:
print(
'obsolete default template detected, update to the latest template options.'
)
with open(template_file, 'w') as f:
f.write(TEMPLATE)
template_file = ut.update_template_file(template_file, current_dict)
else:
print('default template file exists: ' + template_file)
return template_file
def read_template2inps(template_file, inps=None):
"""Read input template options into Namespace inps"""
if not inps:
inps = cmd_line_parse()
inpsDict = vars(inps)
print('read options from template file: '+os.path.basename(template_file))
template = readfile.read_template(inps.template_file)
template = ut.check_template_auto_value(template)
# Update inps if key existed in template file
prefix = 'pysar.network.'
keyList = [i for i in list(inpsDict.keys()) if prefix+i in template.keys()]
for key in keyList:
value = template[prefix+key]
if key in ['coherenceBased', 'keepMinSpanTree']:
inpsDict[key] = value
elif value:
if key in ['minCoherence', 'tempBaseMax', 'perpBaseMax']:
inpsDict[key] = float(value)
elif key in ['connNumMax']:
inpsDict[key] = int(value)
elif key in ['maskFile', 'referenceFile']:
inpsDict[key] = value
elif key == 'aoiYX':
tmp = [i.strip() for i in value.split(',')]
sub_y = sorted([int(i.strip()) for i in tmp[0].split(':')])
sub_x = sorted([int(i.strip()) for i in tmp[1].split(':')])
inps.aoi_pix_box = (sub_x[0], sub_y[0], sub_x[1], sub_y[1])
elif key == 'aoiLALO':
tmp = [i.strip() for i in value.split(',')]
sub_lat = sorted([float(i.strip()) for i in tmp[0].split(':')])
sub_lon = sorted([float(i.strip()) for i in tmp[1].split(':')])
inps.aoi_geo_box = (sub_lon[0], sub_lat[1], sub_lon[1], sub_lat[0])
# Check lookup file
if not inps.lookupFile:
print('Warning: no lookup table file found! Can not use '+key+' option without it.')
print('skip this option.')
inps.aoi_pix_box = None
elif key in ['startDate', 'endDate']:
inpsDict[key] = ptime.yyyymmdd(value)
elif key == 'excludeDate':
inpsDict[key] = ptime.yyyymmdd(value.replace(',', ' ').split())
elif key == 'excludeIfgIndex':
inpsDict[key] += value.replace(',', ' ').split()
inps.excludeIfgIndex = read_input_index_list(inps.excludeIfgIndex, stackFile=inps.file)
# Turn reset on if 1) no input options found to drop ifgram AND 2) there is template input
if all(not i for i in [inps.referenceFile, inps.tempBaseMax, inps.perpBaseMax, inps.connNumMax,
inps.excludeIfgIndex, inps.excludeDate, inps.coherenceBased,
inps.startDate, inps.endDate, inps.reset, inps.manual]):
print('No input option found to remove interferogram')
print('Keep all interferograms by enable --reset option')
inps.reset = True
return inps
def read_template2inps(templateFile, inps=None):
"""Update inps with options from templateFile"""
if not inps:
inps = cmd_line_parse()
template = readfile.read_template(templateFile)
template = ut.check_template_auto_value(template)
key = 'pysar.reference.date'
if key in template.keys() and template[key]:
inps.refDate = template[key]
return inps
def read_inps2dict(inps):
print('read options from template file: '+os.path.basename(inps.templateFile))
template = readfile.read_template(inps.templateFile)
template = ut.check_template_auto_value(template)
iDict = vars(inps)
key_prefix = 'isce.'
key_list = [i.split(key_prefix)[1] for i in template.keys() if i.startswith(key_prefix)]
for key in key_list:
iDict[key] = template[key_prefix+key]
return iDict
def read_inps2dict(inps):
"""Read input Namespace object info into inpsDict"""
# Read input info into inpsDict
inpsDict = vars(inps)
inpsDict['PLATFORM'] = None
# Read template file
template = {}
for fname in inps.template_file:
temp = readfile.read_template(fname)
temp = ut.check_template_auto_value(temp)
template.update(temp)
for key, value in template.items():
inpsDict[key] = value
if 'processor' in template.keys():
template['pysar.load.processor'] = template['processor']
prefix = 'pysar.load.'
key_list = [
i.split(prefix)[1] for i in template.keys() if i.startswith(prefix)
]
for key in key_list:
value = template[prefix + key]
if key in ['processor', 'updateMode', 'compression']:
inpsDict[key] = template[prefix + key]
elif value:
inpsDict[prefix + key] = template[prefix + key]
if inpsDict['compression'] == False:
inpsDict['compression'] = None
# PROJECT_NAME --> PLATFORM
if not inpsDict['PROJECT_NAME']:
inpsDict['PROJECT_NAME'] = sensor.project_name2sensor_name(
inps.template_file)[1]
inpsDict['PLATFORM'] = sensor.project_name2sensor_name(
inpsDict['PROJECT_NAME'])[0]
if inpsDict['PLATFORM']:
print('platform : {}'.format(inpsDict['PLATFORM']))
print('processor: {}'.format(inpsDict['processor']))
# Here to insert code to check default file path for miami user
if (auto_path.autoPath and 'SCRATCHDIR' in os.environ
and inpsDict['PROJECT_NAME'] is not None):
print(('check auto path setting for Univ of Miami users'
' for processor: {}'.format(inpsDict['processor'])))
inpsDict = auto_path.get_auto_path(
processor=inpsDict['processor'],
project_name=inpsDict['PROJECT_NAME'],
template=inpsDict)
return inpsDict
def read_template2inps(template_file, inps=None):
"""Read input template options into Namespace inps"""
if not inps:
inps = cmd_line_parse()
inpsDict = vars(inps)
print('read options from tempalte file: '+os.path.basename(inps.template_file))
template = readfile.read_template(template_file)
template = ut.check_template_auto_value(template)
key_list = [i for i in list(inpsDict.keys()) if key_prefix+i in template.keys()]
for key in key_list:
value = template[key_prefix+key]
if value:
if key in ['waterMaskFile']:
inpsDict[key] = value
return inps
def read_input_attribute(argv, print_msg=True):
atr_new = dict()
for i in range(1, len(argv)):
if os.path.isfile(argv[i]):
atr_tmp = readfile.read_template(argv[i])
atr_new.update(atr_tmp)
else:
atr_tmp = argv[i].split('=')
atr_new[atr_tmp[0].strip()] = atr_tmp[1].strip()
if print_msg:
print(
"The following attributes will be added/updated, or removed if new value is 'None':"
)
info.print_attributes(atr_new)
return atr_new
def create_custom_template(custom_template_file, work_dir):
""" Creates or restores custom template file. """
if custom_template_file:
# Copy custom template file to work directory
if utils.run_or_skip(os.path.basename(custom_template_file),
custom_template_file,
check_readable=False) == 'run':
shutil.copy2(custom_template_file, work_dir)
# Read custom template
logger.log(
loglevel.INFO,
'read custom template file: {}'.format(custom_template_file))
return readfile.read_template(custom_template_file)
else:
return dict()
def read_template2inps(template_file, inps=None):
"""Read input template options into Namespace inps"""
if not inps:
inps = cmd_line_parse()
print('read options from tempalte file: ' + os.path.basename(inps.template_file))
template = readfile.read_template(inps.template_file)
prefix = 'pysar.unwrapError.'
key = prefix+'method'
if key in template.keys():
value = template[key]
if value in ['bridging', 'phase_closure']:
inps.method = value
elif value not in ['auto', 'no']:
inps.method = None
else:
print('Unrecognized input for %s: %s' % (key, value))
key = prefix+'maskFile'
if key in template.keys():
value = template[key]
if value not in ['auto', 'no']:
inps.mask_file = value
key = prefix+'yx'
if key in template.keys():
value = template[key]
if value not in ['auto', 'no']:
yx = value.replace(';', ' ').replace(',', ' ').split()
yx = [int(i) for i in yx]
inps.y = yx[0::2]
inps.x = yx[1::2]
key = prefix+'ramp'
if key in template.keys():
value = template[key]
if value in ['auto']:
inps.ramp_type = 'plane'
elif value in ['plane', 'quadratic']:
inps.ramp_type = value
else:
print('Unrecognized input for %s: %s' % (key, value))
return inps
def read_template2inps(templateFile, inps):
"""Update inps with pysar.residualRms.* option from templateFile"""
if not inps:
inps = cmd_line_parse()
inpsDict = vars(inps)
print('read options from template file: '+os.path.basename(templateFile))
template = readfile.read_template(templateFile)
template = ut.check_template_auto_value(template)
prefix = 'pysar.residualRms.'
keyList = [i for i in list(inpsDict.keys()) if prefix+i in template.keys()]
for key in keyList:
value = template[prefix+key]
if value:
if key in ['maskFile', 'ramp']:
inpsDict[key] = value
elif key in ['cutoff']:
inpsDict[key] = float(value)
return inps
def read_subset_template2box(template_file):
"""Read pysar.subset.lalo/yx option from template file into box type
Return None if not specified.
"""
tmpl = readfile.read_template(template_file)
try:
sub = [i.strip() for i in tmpl['pysar.subset.lalo'].split(',')]
sub_lat = sorted([float(i.strip()) for i in sub[0].split(':')])
sub_lon = sorted([float(i.strip()) for i in sub[1].split(':')])
geo_box = (sub_lon[0], sub_lat[1], sub_lon[1], sub_lat[0])
except:
geo_box = None
try:
sub = [i.strip() for i in tmpl['pysar.subset.yx'].split(',')]
sub_y = sorted([int(i.strip()) for i in sub[0].split(':')])
sub_x = sorted([int(i.strip()) for i in sub[1].split(':')])
pix_box = (sub_x[0], sub_y[0], sub_x[1], sub_y[1])
except:
pix_box = None
return pix_box, geo_box
def read_template2inps(template_file, inps):
"""Read input template options into Namespace inps"""
if not inps:
inps = cmd_line_parse()
inpsDict = vars(inps)
template = readfile.read_template(template_file)
template = ut.check_template_auto_value(template)
keyList = [
i for i in list(inpsDict.keys()) if key_prefix + i in template.keys()
]
for key in keyList:
value = template[key_prefix + key]
if key in ['maskDataset']:
inpsDict[key] = value
elif value:
if key in ['maskThreshold']:
inpsDict[key] = float(value)
elif key in ['weightFunc', 'residualNorm', 'waterMaskFile']:
inpsDict[key] = value
return inps
def read_template2inps(template_file, inps=None):
"""Read input template file into inps.excludeDate"""
if not inps:
inps = cmd_line_parse()
inpsDict = vars(inps)
print('read options from template file: '+os.path.basename(template_file))
template = readfile.read_template(inps.template_file)
template = ut.check_template_auto_value(template)
# Read template option
keyList = [i for i in list(inpsDict.keys()) if key_prefix+i in template.keys()]
for key in keyList:
value = template[key_prefix+key]
if key in ['phaseVelocity']:
inpsDict[key] = value
elif value:
if key in ['polyOrder']:
inpsDict[key] = int(value)
elif key in ['excludeDate','stepFuncDate']:
inpsDict[key] = ptime.yyyymmdd(value.replace(',', ' ').split())
return inps
def read_template2inps(template_file, inps=None):
"""Read input template options into Namespace inps"""
if not inps:
inps = cmd_line_parse()
print('read options from template file: ' +
os.path.basename(template_file))
template = readfile.read_template(template_file)
# Coherence-based network modification
prefix = 'pysar.save.hdfEos5.'
key = prefix + 'update'
if key in template.keys() and template[key] == 'yes':
inps.update = True
key = prefix + 'subset'
if key in template.keys() and template[key] == 'yes':
inps.subset = True
return inps
def read_template2inps(template_file, inps=None):
"""Read input template file into inps.ex_date"""
if not inps:
inps = cmd_line_parse()
template = readfile.read_template(template_file)
# Read template option
prefix = 'pysar.topographicResidual.'
key = prefix+'polyOrder'
if key in template.keys():
value = template[key]
if value == 'auto':
inps.poly_order = 2
else:
inps.poly_order = int(value)
key = prefix+'excludeDate'
if key in template.keys():
value = template[key]
if value not in ['auto', 'no']:
value = value.replace(',', ' ').split()
value = ptime.yyyymmdd(value)
inps.ex_date += value
key = prefix+'stepFuncDate'
if key in template.keys():
value = template[key]
if value not in ['auto', 'no']:
value = value.replace(',', ' ').split()
value = ptime.yyyymmdd(value)
inps.step_date += value
key = prefix+'phaseVelocity'
if key in template.keys():
value = template[key]
if value.lower() not in ['auto', 'no']:
inps.min_phase_velocity = True
return inps
def read_template2inps(template_file, inps=None):
"""Read input template options into Namespace inps"""
if not inps:
inps = cmd_line_parse()
inpsDict = vars(inps)
print('read options from template file: ' +
os.path.basename(template_file))
template = readfile.read_template(inps.template_file)
template = ut.check_template_auto_value(template)
# Coherence-based network modification
prefix = 'pysar.network.'
key = prefix + 'maskFile'
if key in template.keys():
if template[key]:
inps.maskFile = template[key]
key = prefix + 'minCoherence'
if key in template.keys():
if template[key]:
inps.coh_thres = float(template[key])
return inps
def check_template_auto_value(templateDict, auto_file='../defaults/pysarApp.cfg'):
"""Replace auto value based on $PYSAR_HOME/pysar/defaults/template.cfg file."""
# Read default template value and turn yes/no to True/False
templateAutoFile = os.path.join(os.path.dirname(__file__), auto_file)
templateAutoDict = readfile.read_template(templateAutoFile)
# Update auto value of input template dict
for key, value in templateDict.items():
if value == 'auto' and key in templateAutoDict.keys():
templateDict[key] = templateAutoDict[key]
# Change yes --> True and no --> False
specialValues = {'yes': True,
'True': True,
'no': False,
'False': False,
'none': None,
}
for key, value in templateDict.items():
if value in specialValues.keys():
templateDict[key] = specialValues[value]
return templateDict
def _read_template(self):
# read custom template, to:
# 1) update default template
# 2) add metadata to ifgramStack file and HDF-EOS5 file
self.customTemplate = None
if self.customTemplateFile:
cfile = self.customTemplateFile
# Copy custom template file to INPUTS directory for backup
inputs_dir = os.path.join(self.workDir, 'INPUTS')
if not os.path.isdir(inputs_dir):
os.makedirs(inputs_dir)
print('create directory:', inputs_dir)
if ut.run_or_skip(out_file=os.path.join(inputs_dir,
os.path.basename(cfile)),
in_file=cfile,
check_readable=False) == 'run':
shutil.copy2(cfile, inputs_dir)
print('copy {} to INPUTS directory for backup.'.format(
os.path.basename(cfile)))
# Read custom template
print('read custom template file:', cfile)
cdict = readfile.read_template(cfile)
# correct some loose type errors
standardValues = {
'def': 'auto',
'default': 'auto',
'y': 'yes',
'on': 'yes',
'true': 'yes',
'n': 'no',
'off': 'no',
'false': 'no'
}
for key, value in cdict.items():
if value in standardValues.keys():
cdict[key] = standardValues[value]
for key in ['pysar.deramp', 'pysar.troposphericDelay.method']:
if key in cdict.keys():
cdict[key] = cdict[key].lower().replace('-', '_')
if 'processor' in cdict.keys():
cdict['pysar.load.processor'] = cdict['processor']
# these metadata are used in load_data.py only, not needed afterwards
# (in order to manually add extra offset when the lookup table is shifted)
# (seen in ROI_PAC product sometimes)
for key in ['SUBSET_XMIN', 'SUBSET_YMIN']:
if key in cdict.keys():
cdict.pop(key)
self.customTemplate = dict(cdict)
# Update default template file based on custom template
print('update default template based on input custom template')
self.templateFile = ut.update_template_file(
self.templateFile, self.customTemplate)
print('read default template file:', self.templateFile)
self.template = readfile.read_template(self.templateFile)
self.template = ut.check_template_auto_value(self.template)
# correct some loose setup conflicts
if self.template['pysar.geocode'] is False:
for key in ['pysar.save.hdfEos5', 'pysar.save.kmz']:
if self.template[key] is True:
self.template['pysar.geocode'] = True
print('Turn ON pysar.geocode in order to run {}.'.format(
key))
break
return
def read_template2inps(templateFile, inps=None):
"""Read network options from template file into Namespace variable inps"""
if not inps:
inps = cmd_line_parse()
inpsDict = vars(inps)
# Read template file
template = readfile.read_template(templateFile)
template = ut.check_template_auto_value(
template, auto_file='../defaults/selectNetwork.cfg')
if not template:
log('Empty template: ' + templateFile)
return None
prefix = 'selectNetwork.'
# Check obsolete option prefix
for i in ['selectPairs.', 'select.network.']:
if any(i in key for key in template.keys()):
msg = 'obsolete option prefix detected: {}\n'.format(i)
msg += 'Use {} instead'.format(prefix)
raise Exception(msg)
if all(prefix not in key for key in template.keys()):
msg = 'no valid input option deteced in template file!\n'
msg += 'Check the template below for supported options:\n'
msg += TEMPLATE
raise Exception(msg)
# convert template into inpsDict
keyList = [
i for i in list(inpsDict.keys()) if prefix + i in template.keys()
]
for key in keyList:
value = template[prefix + key]
# bool
if key in ['keepSeasonal']:
inpsDict[key] = value
elif value:
# str
if key in ['method', 'referenceFile', 'tempPerpList']:
inpsDict[key] = value
# date in YYYYMMDD
elif key in ['masterDate', 'startDate', 'endDate']:
inpsDict[key] = ptime.yyyymmdd(value)
# list of dates in YYYYMMDD
elif key in ['excludeDate']:
inps.excludeDate = ptime.yyyymmdd(
[i.strip() for i in value.split(',')])
# float
elif key in [
'perpBaseMax', 'tempBaseMax', 'tempBaseMin',
'dopOverlapMin'
]:
inpsDict[key] = float(value)
# int
elif key in ['connNum']:
inpsDict[key] = int(value)
# read tempPerpList from str
if isinstance(inps.tempPerpList, str):
inps.tempPerpList = [[float(j) for j in i.split(',')]
for i in inps.tempPerpList.split(';')]
# Initial network using input methods
inps.method = inps.method.lower().replace('-', '_')
if inps.method in ['star', 'ps']:
inps.method = 'star'
elif inps.method.startswith('seq'):
inps.method = 'sequential'
elif inps.method.startswith('hierar'):
inps.method = 'hierarchical'
elif inps.method in ['mst', 'min_spanning_tree', 'minimum_spanning_tree']:
inps.method = 'mst'
# for coherence prediction
key = 'PLATFORM'
if key in template.keys() and not inps.sensor:
inps.sensor = template[key]
key = 'COH_COLOR_JUMP'
if key in template.keys():
inps.coh_thres = float(template[key])
# project name and sensor
project_name = os.path.splitext(os.path.basename(inps.template_file))[0]
log('project name: ' + project_name)
if not inps.sensor:
inps.sensor = sensor.project_name2sensor(project_name)[0]
# Output directory/filename
if not inps.outfile:
if autoPath and 'SCRATCHDIR' in os.environ:
inps.out_dir = os.getenv(
'SCRATCHDIR') + '/' + project_name + '/PROCESS'
else:
try:
inps.out_dir = os.path.dirname(
os.path.abspath(inps.referenceFile))
except:
inps.out_dir = os.path.dirname(
os.path.abspath(inps.baseline_file))
inps.outfile = inps.out_dir + '/ifgram_list.txt'
# Auto path of bl_list.txt file (for Miami user)
if not inps.baseline_file and autoPath and 'SCRATCHDIR' in os.environ:
bl_file = os.path.join(os.getenv('SCRATCHDIR'),
'{}/SLC/bl_list.txt'.format(project_name))
if os.path.isfile(bl_file):
inps.baseline_file = bl_file
if not inps.referenceFile and not inps.baseline_file:
raise Exception(
'No baseline file or reference file found! At least one is required.'
)
return inps
def read_template(inps):
print('\n********** Read Template File **********')
# default template
inps.templateFile = os.path.join(inps.workDir, 'pysarApp_template.txt')
if not os.path.isfile(inps.templateFile):
print('generate default template file:', inps.templateFile)
shutil.copy2(inps.autoTemplateFile, inps.workDir)
else:
check_obsolete_default_template(inps)
# custom template
customTemplate = None
if inps.customTemplateFile:
# Copy custom template file to work directory
inputs_dir = os.path.join(inps.workDir, 'INPUTS')
if ut.run_or_skip(out_file=os.path.join(inputs_dir, os.path.basename(inps.customTemplateFile)),
in_file=inps.customTemplateFile,
check_readable=False) == 'run':
if not os.path.isdir(inputs_dir):
os.makedirs(inputs_dir)
print('create directory:', inputs_dir)
shutil.copy2(inps.customTemplateFile, inputs_dir)
print('copy {} to INPUTS directory'.format(os.path.basename(inps.customTemplateFile)))
# Read custom template
print('read custom template file:', inps.customTemplateFile)
customTemplate = readfile.read_template(inps.customTemplateFile)
# correct some loose type errors
standardValues = {'def':'auto', 'default':'auto',
'y':'yes', 'on':'yes', 'true':'yes',
'n':'no', 'off':'no', 'false':'no'
}
for key, value in customTemplate.items():
if value in standardValues.keys():
customTemplate[key] = standardValues[value]
for key in ['pysar.deramp', 'pysar.troposphericDelay.method']:
if key in customTemplate.keys():
customTemplate[key] = customTemplate[key].lower().replace('-', '_')
if 'processor' in customTemplate.keys():
customTemplate['pysar.load.processor'] = customTemplate['processor']
for key in ['SUBSET_XMIN', 'SUBSET_YMIN']:
if key in customTemplate.keys():
customTemplate.pop(key)
# Update default template with custom input template
print('update default template based on input custom template')
inps.templateFile = ut.update_template_file(inps.templateFile, customTemplate)
if inps.generate_template:
raise SystemExit('Exit as planned after template file generation.')
print('read default template file:', inps.templateFile)
template = readfile.read_template(inps.templateFile)
template = ut.check_template_auto_value(template)
# Get existing files name: unavco_attributes.txt
try:
inps.unavcoMetadataFile = ut.get_file_list('unavco_attribute*txt', abspath=True)[0]
except:
inps.unavcoMetadataFile = None
print('No UNAVCO attributes file found.')
return inps, template, customTemplate
def main(argv):
messageRsmas.log(' '.join(argv))
inps = dem_parser()
custom_template = readfile.read_template(inps.custom_template_file)
cwd = make_dem_dir()
if 'sentinelStack.demMethod' not in list(custom_template.keys()):
custom_template['sentinelStack.demMethod'] = '?'
if custom_template['sentinelStack.demMethod'] == 'bbox' or custom_template[
'sentinelStack.demMethod'] == 'isce' or inps.isce:
print('You hace started isce')
bbox = custom_template['sentinelStack.boundingBox']
south = bbox.split(' ')[0].split('\'')[
1] # assumes quotes '-1 0.15 -91.3 -91.0'
north = bbox.split(' ')[1]
west = bbox.split(' ')[2]
east = bbox.split(' ')[3].split('\'')[0]
south = round(float(south) - 0.5)
north = round(float(north) + 0.5)
west = round(float(west) - 0.5)
east = round(float(east) + 0.5)
demBbox = str(int(south)) + ' ' + str(int(north)) + ' ' + str(
int(west)) + ' ' + str(int(east))
cmd = 'dem.py -a stitch -b ' + demBbox + ' -c -u https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11/'
messageRsmas.log(cmd)
try:
output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT,
shell=True,
universal_newlines=True)
except subprocess.CalledProcessError as exc:
print("Command failed. Exit code, StdErr:", exc.returncode,
exc.output)
sys.exit('Error produced by dem.py')
else:
if 'Could not create a stitched DEM. Some tiles are missing' in output:
os.chdir('..')
shutil.rmtree('DEM')
sys.exit('Error in dem.py: Tiles are missing. Ocean???')
xmlFile = glob.glob('demLat_*.wgs84.xml')[0]
fin = open(xmlFile, 'r')
fout = open("tmp.txt", "wt")
for line in fin:
fout.write(line.replace('demLat', cwd + '/demLat'))
fin.close()
fout.close()
os.rename('tmp.txt', xmlFile)
elif custom_template['sentinelStack.demMethod'] == 'ssara' or inps.ssara:
call_ssara_dem(custom_template, inps, cwd)
print('You have finished SSARA!')
else:
sys.ext('Error unspported demMethod option: ' +
custom_template['sentinelStack.demMethod'])
print('\n###############################################')
print('End of dem_rsmas.py')
print('################################################\n')
def main(argv):
messageRsmas.log(' '.join(argv))
inps = dem_parser()
custom_template = readfile.read_template(inps.custom_template_file)
cwd = make_dem_dir()
# can sentinelStack.demMethod be removed? I think parser is the replacement
if 'sentinelStack.demMethod' not in custom_template.keys():
custom_template['sentinelStack.demMethod'] = 'bbox'
if custom_template['sentinelStack.demMethod'] == 'bbox':
bbox = custom_template['sentinelStack.boundingBox']
south = bbox.split(' ')[0].split('\'')[
1] # assumes quotes '-1 0.15 -91.3 -91.0'
north = bbox.split(' ')[1]
west = bbox.split(' ')[2]
east = bbox.split(' ')[3].split('\'')[0]
elif custom_template['sentinelStack.demMethod'] == 'ssara':
call_ssara_dem(custom_template, inps)
cmd = 'ssara_federated_query.py ' + custom_template[
'ssaraopt'] + ' --dem'
output = subprocess.check_output(cmd, shell=True)
output = output.split("\n")
for line in output:
if line.startswith("wget"):
coordList = line.split("?")[1].split("&")[0:4]
for item in coordList:
if "north" in item:
north = item.split("=")[1]
if "south" in item:
south = item.split("=")[1]
if "east" in item:
east = item.split("=")[1]
if "west" in item:
west = item.split("=")[1]
else:
sys.ext('Error unspported demMethod option: ' +
custom_template['sentinelStack.demMethod'])
if inps.ssara:
call_ssara_dem(custom_template, inps)
print('####### CONTINUED')
else:
print('not ssara')
south = round(float(south) - 0.5)
north = round(float(north) + 0.5)
west = round(float(west) - 0.5)
east = round(float(east) + 0.5)
demBbox = str(int(south)) + ' ' + str(int(north)) + ' ' + str(
int(west)) + ' ' + str(int(east))
cmd = 'dem.py -a stitch -b ' + demBbox + ' -c -u https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11/'
messageRsmas.log(cmd)
try:
output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT,
shell=True,
universal_newlines=True)
except subprocess.CalledProcessError as exc:
print("Command failed. Exit code, StdErr:", exc.returncode, exc.output)
sys.exit('Error produced by dem.py')
else:
#print("Success. StdOut \n{}\n".format(output))
if 'Could not create a stitched DEM. Some tiles are missing' in output:
os.chdir('..')
shutil.rmtree('DEM')
sys.exit('Error in dem.py: Tiles are missing. Ocean???')
xmlFile = glob.glob('demLat_*.wgs84.xml')[0]
fin = open(xmlFile, 'r')
fout = open("tmp.txt", "wt")
for line in fin:
fout.write(line.replace('demLat', cwd + '/demLat'))
fin.close()
fout.close()
os.rename('tmp.txt', xmlFile)
print '\n###############################################'
print 'End of dem_rsmas.py'
print '################################################\n'
def read_stripmap_baseline(baseline_file):
fDict = readfile.read_template(baseline_file, delimiter=' ')
bperp_top = float(fDict['PERP_BASELINE_TOP'])
bperp_bottom = float(fDict['PERP_BASELINE_BOTTOM'])
return [bperp_top, bperp_bottom]