def startup(self): """The starting point of the workflow. It runs everytime. It 1) grab project name if given 2) grab and go to work directory 3) get and read template(s) options """ #1. Get projectName self.projectName = None if self.customTemplateFile: self.projectName = os.path.splitext( os.path.basename(self.customTemplateFile))[0] print('Project name:', self.projectName) #2. Go to the work directory #2.1 Get workDir if not self.workDir: if autoPath and 'SCRATCHDIR' in os.environ and self.projectName: self.workDir = os.path.join(os.getenv('SCRATCHDIR'), self.projectName, 'mintpy') else: self.workDir = os.getcwd() self.workDir = os.path.abspath(self.workDir) #2.2 Go to workDir if not os.path.isdir(self.workDir): os.makedirs(self.workDir) print('create directory:', self.workDir) os.chdir(self.workDir) print("Go to work directory:", self.workDir) #3. Read templates #3.1 Get default template file lfile = os.path.join(os.path.dirname(__file__), 'defaults/smallbaselineApp.cfg') #latest version cfile = os.path.join(self.workDir, 'smallbaselineApp.cfg') #current version if not os.path.isfile(cfile): print('copy default template file {} to work directory'.format( lfile)) shutil.copy2(lfile, self.workDir) else: #cfile is obsolete if any key is missing ldict = readfile.read_template(lfile) cdict = readfile.read_template(cfile) if any([key not in cdict.keys() for key in ldict.keys()]): print( 'obsolete default template detected, update to the latest version.' ) shutil.copy2(lfile, self.workDir) #keep the existing option value from obsolete template file ut.update_template_file(cfile, cdict) self.templateFile = cfile # 3.2 read (custom) template files into dicts self._read_template() return
def startup(self): """The starting point of the workflow. It runs everytime. It 1) grab project name if given 2) grab and go to work directory 3) get and read template(s) options """ #1. Get projectName self.projectName = None if self.customTemplateFile: self.projectName = os.path.splitext(os.path.basename(self.customTemplateFile))[0] print('Project name:', self.projectName) #2. Go to the work directory #2.1 Get workDir if not self.workDir: if autoPath and 'SCRATCHDIR' in os.environ and self.projectName: self.workDir = os.path.join(os.getenv('SCRATCHDIR'), self.projectName, 'mintpy') else: self.workDir = os.getcwd() self.workDir = os.path.abspath(self.workDir) #2.2 Go to workDir if not os.path.isdir(self.workDir): os.makedirs(self.workDir) print('create directory:', self.workDir) os.chdir(self.workDir) print("Go to work directory:", self.workDir) #3. Read templates #3.1 Get default template file lfile = os.path.join(os.path.dirname(__file__), 'defaults/smallbaselineApp.cfg') #latest version cfile = os.path.join(self.workDir, 'smallbaselineApp.cfg') #current version if not os.path.isfile(cfile): print('copy default template file {} to work directory'.format(lfile)) shutil.copy2(lfile, self.workDir) else: #cfile is obsolete if any key is missing ldict = readfile.read_template(lfile) cdict = readfile.read_template(cfile) if any([key not in cdict.keys() for key in ldict.keys()]): print('obsolete default template detected, update to the latest version.') shutil.copy2(lfile, self.workDir) #keep the existing option value from obsolete template file ut.update_template_file(cfile, cdict) self.templateFile = cfile # 3.2 read (custom) template files into dicts self._read_template() return
def read_template2inps(template_file, inps): """Read input template options into Namespace inps""" print('read input option from template file: ' + template_file) if not inps: inps = cmd_line_parse() inps_dict = vars(inps) template = readfile.read_template(template_file) template = ut.check_template_auto_value(template) prefix = 'mintpy.geocode.' key_list = [i for i in list(inps_dict.keys()) if prefix + i in template.keys()] for key in key_list: value = template[prefix + key] if value: if key == 'SNWE': inps_dict[key] = tuple([float(i) for i in value.split(',')]) elif key in ['latStep', 'lonStep']: inps_dict[key] = float(value) elif key in ['interpMethod']: inps_dict[key] = value elif key == 'fillValue': if 'nan' in value.lower(): inps_dict[key] = np.nan else: inps_dict[key] = float(value) inps.laloStep = [inps.latStep, inps.lonStep] if None in inps.laloStep: inps.laloStep = None return inps
def main(argv): parser = build_parser() parseArgs = parser.parse_args() username = parseArgs.user password = parseArgs.password host = parseArgs.host db = parseArgs.db working_dir = parseArgs.folder # make sure we have a final / so the below code doesn't break if working_dir[-1] != "/": working_dir += "/" unavco_name = parseArgs.unavco_name attributes_file = working_dir + "add_Attribute.txt" attributes = readfile.read_template(attributes_file) dbController = InsarDatabaseController(username, password, host, db) dbController.connect() for key in attributes: print("Setting attribute " + key + " to " + attributes[key]) if key == "plotAttributes": dbController.add_plot_attribute(unavco_name, key, attributes[key]) else: dbController.add_attribute(unavco_name, key, attributes[key]) dbController.index_table_on("extra_attributes", "area_id", "area_id_idx") dbController.close()
def update_template_file(template_file, extra_dict): """Update option value in template_file with value from input extra_dict""" # Compare and skip updating template_file if no new option value found. update = False orig_dict = readfile.read_template(template_file) for key, value in orig_dict.items(): if key in extra_dict.keys() and extra_dict[key] != value: update = True if not update: print('No new option value found, skip updating ' + template_file) return template_file # Update template_file with new value from extra_dict tmp_file = template_file + '.tmp' f_tmp = open(tmp_file, 'w') for line in open(template_file, 'r'): c = [i.strip() for i in line.strip().split('=', 1)] if not line.startswith(('%', '#')) and len(c) > 1: key = c[0] value = str.replace(c[1], '\n', '').split("#")[0].strip() if key in extra_dict.keys() and extra_dict[key] != value: line = line.replace(value, extra_dict[key], 1) print(' {}: {} --> {}'.format(key, value, extra_dict[key])) f_tmp.write(line) f_tmp.close() # Overwrite exsting original template file mvCmd = 'mv {} {}'.format(tmp_file, template_file) os.system(mvCmd) return template_file
def update_template_file(template_file, extra_dict, delimiter='='): """Update option value in template_file with value from input extra_dict""" # Compare and skip updating template_file if no new option value found. update = False orig_dict = readfile.read_template(template_file) for key, value in orig_dict.items(): if key in extra_dict.keys() and extra_dict[key] != value: update = True if not update: print('No new option value found, skip updating ' + template_file) return template_file # Update template_file with new value from extra_dict tmp_file = template_file + '.tmp' f_tmp = open(tmp_file, 'w') for line in open(template_file, 'r'): c = [i.strip() for i in line.strip().split(delimiter, 1)] if not line.startswith(('%', '#')) and len(c) > 1: key = c[0] value = str.replace(c[1], '\n', '').split("#")[0].strip() if key in extra_dict.keys() and extra_dict[key] != value: # use "= {OLD_VALUE}" for search/replace to be more robust # against the scenario when key name contains {OLD_VALUE} # i.e. mintpy.load.autoPath old_value_str = re.findall(delimiter + '[\s]*' + value, line)[0] new_value_str = old_value_str.replace(value, extra_dict[key]) line = line.replace(old_value_str, new_value_str, 1) print(' {}: {} --> {}'.format(key, value, extra_dict[key])) f_tmp.write(line) f_tmp.close() # Overwrite exsting original template file shutil.move(tmp_file, template_file) return template_file
def read_template2inps(template_file, inps): """Read input template options into Namespace inps""" print('read input option from template file: ' + template_file) if not inps: inps = cmd_line_parse() inps_dict = vars(inps) template = readfile.read_template(template_file) template = ut.check_template_auto_value(template) prefix = 'mintpy.geocode.' key_list = [ i for i in list(inps_dict.keys()) if prefix + i in template.keys() ] for key in key_list: value = template[prefix + key] if value: if key == 'SNWE': inps_dict[key] = tuple([float(i) for i in value.split(',')]) elif key in ['latStep', 'lonStep']: inps_dict[key] = float(value) elif key in ['interpMethod']: inps_dict[key] = value elif key == 'fillValue': if 'nan' in value.lower(): inps_dict[key] = np.nan else: inps_dict[key] = float(value) inps.laloStep = [inps.latStep, inps.lonStep] if None in inps.laloStep: inps.laloStep = None return inps
def get_snwe(inps, min_buffer=2, step=10): # get bounding box if inps.SNWE: SNWE = inps.SNWE else: custom_template = readfile.read_template(inps.template_file) if ('weather.boundingBox' in custom_template): SNWE = custom_template['weather.boundingBox'].split(' ') else: SNWE = custom_template['topsStack.boundingBox'].split(' ') lat0 = float(SNWE[0]) lat1 = float(SNWE[1]) lon0 = float(SNWE[2]) lon1 = float(SNWE[3]) # lat/lon0/1 --> SNWE S = np.floor(min(lat0, lat1) - min_buffer).astype(int) N = np.ceil(max(lat0, lat1) + min_buffer).astype(int) W = np.floor(min(lon0, lon1) - min_buffer).astype(int) E = np.ceil(max(lon0, lon1) + min_buffer).astype(int) # SNWE in multiple of 10 if step > 1: S = tropo_pyaps3.floor2multiple(S, step=step) W = tropo_pyaps3.floor2multiple(W, step=step) N = tropo_pyaps3.ceil2multiple(N, step=step) E = tropo_pyaps3.ceil2multiple(E, step=step) return (S, N, W, E)
def read_template2inps(template_file, inps): """Read input template options into Namespace inps""" print('read input option from template file: ' + template_file) if not inps: inps = cmd_line_parse() inps_dict = vars(inps) template = readfile.read_template(template_file) template = ut.check_template_auto_value(template) prefix = 'mintpy.geocode.' key_list = [ i for i in list(inps_dict.keys()) if prefix + i in template.keys() ] for key in key_list: value = template[prefix + key] if value: if key in ['SNWE', 'laloStep']: value = value.replace('[', '').replace(']', '').replace(',', ' ') inps_dict[key] = [float(i) for i in value.split()] elif key in ['interpMethod']: inps_dict[key] = value elif key == 'fillValue': if 'nan' in value.lower(): inps_dict[key] = np.nan else: inps_dict[key] = float(value) # computing configurations key = 'mintpy.compute.maxMemory' if key in template.keys() and template[key]: inps.maxMemory = float(template[key]) return inps
def check_template_auto_value(templateDict, auto_file='defaults/smallbaselineApp_auto.cfg'): """Replace auto value based on the input auto config file.""" ## Read default template value and turn yes/no to True/False templateAutoFile = os.path.join(os.path.dirname(mintpy.__file__), auto_file) templateAutoDict = readfile.read_template(templateAutoFile) # if cluster != local, change auto value of numWorker cluster_key = 'mintpy.compute.cluster' cluster = templateDict.get(cluster_key, 'auto').lower() if cluster == 'auto': cluster = templateAutoDict[cluster_key] if cluster != 'local': templateAutoDict['mintpy.compute.numWorker'] = '40' ## Update auto value of input template dict for key, value in templateDict.items(): if value == 'auto' and key in templateAutoDict.keys(): templateDict[key] = templateAutoDict[key] # Change yes --> True, no --> False and none --> None specialValues = {'yes' : True, 'true' : True, 'no' : False, 'false': False, 'none' : None, } for key, value in templateDict.items(): value = value.lower() if value in specialValues.keys(): templateDict[key] = specialValues[value] return templateDict
def update_template_file(template_file, extra_dict): """Update option value in template_file with value from input extra_dict""" # Compare and skip updating template_file if no new option value found. update = False orig_dict = readfile.read_template(template_file) for key, value in orig_dict.items(): if key in extra_dict.keys() and extra_dict[key] != value: update = True if not update: print('No new option value found, skip updating '+template_file) return template_file # Update template_file with new value from extra_dict tmp_file = template_file+'.tmp' f_tmp = open(tmp_file, 'w') for line in open(template_file, 'r'): c = [i.strip() for i in line.strip().split('=', 1)] if not line.startswith(('%', '#')) and len(c) > 1: key = c[0] value = str.replace(c[1], '\n', '').split("#")[0].strip() if key in extra_dict.keys() and extra_dict[key] != value: line = line.replace(value, extra_dict[key], 1) print(' {}: {} --> {}'.format(key, value, extra_dict[key])) f_tmp.write(line) f_tmp.close() # Overwrite exsting original template file mvCmd = 'mv {} {}'.format(tmp_file, template_file) os.system(mvCmd) return template_file
def configure(self): inps = cmd_line_parse(self.iargs) # read network info inps = read_network_info(inps) # copy inps to self object for key, value in inps.__dict__.items(): setattr(self, key, value) # auto figure size if not self.fig_size: ds_shape = readfile.read(self.img_file)[0].shape fig_size = pp.auto_figure_size(ds_shape, disp_cbar=True, ratio=0.7) self.fig_size = [fig_size[0]+fig_size[1], fig_size[1]] vprint('create figure in size of {} inches'.format(self.fig_size)) # read aux data # 1. temporal coherence value self.tcoh = None if self.tcoh_file: self.tcoh = readfile.read(self.tcoh_file)[0] # 2. minimum used coherence from template file self.min_coh_used = 0.0 if self.template_file: template = readfile.read_template(self.template_file) template = ut.check_template_auto_value(template) if template['mintpy.networkInversion.maskDataset'] == 'coherence': self.min_coh_used = float(template['mintpy.networkInversion.maskThreshold']) vprint('Pixel-wised masking is applied in invert_network step') return
def read_template2inps(template_file, inps=None): """Read input template file into inps.excludeDate""" if not inps: inps = cmd_line_parse() iDict = vars(inps) print('read options from template file: ' + os.path.basename(template_file)) template = readfile.read_template(inps.template_file) template = ut.check_template_auto_value(template) # Read template option prefix = 'mintpy.velocity.' keyList = [i for i in list(iDict.keys()) if prefix + i in template.keys()] for key in keyList: value = template[prefix + key] if key in ['bootstrap']: iDict[key] = value if value: if key in ['startDate', 'endDate']: iDict[key] = ptime.yyyymmdd(value) elif key in ['excludeDate']: value = value.replace('[', '').replace(']', '').replace(',', ' ') iDict[key] = ptime.yyyymmdd(value.split()) elif key in ['bootstrapCount']: iDict[key] = int(value) key = 'mintpy.compute.maxMemory' if key in template.keys() and template[key]: inps.maxMemory = float(template[key]) return inps
def read_template_file2inps(template_file, inps=None): """Read seed/reference info from template file and update input namespace""" if not inps: inps = cmd_line_parse(['']) inps_dict = vars(inps) template = readfile.read_template(template_file) template = ut.check_template_auto_value(template) prefix = 'mintpy.reference.' key_list = [i for i in list(inps_dict) if prefix + i in template.keys()] for key in key_list: value = template[prefix + key] if value: if key in ['coherenceFile', 'maskFile']: inps_dict[key] = value elif key == 'minCoherence': inps_dict[key] = float(value) key = prefix + 'yx' if key in template.keys(): value = template[key] if value: inps.ref_y, inps.ref_x = [int(i) for i in value.split(',')] key = prefix + 'lalo' if key in template.keys(): value = template[key] if value: inps.ref_lat, inps.ref_lon = [float(i) for i in value.split(',')] return inps
def read_template2inps(template_file, inps=None): """Read input template file into inps""" if not inps: inps = cmd_line_parse() iDict = vars(inps) print('read options from template file: {}'.format( os.path.basename(template_file))) template = readfile.read_template(template_file) template = ut.check_template_auto_value(template) # ignore template options with default auto values # so that options from input arguments have higher priority # than template options with auto values for key in list(template.keys()): if template[key] == 'auto': template.pop(key) # pass options from template to inps key_prefix = 'mintpy.load.' keys = [i for i in list(iDict.keys()) if key_prefix + i in template.keys()] for key in keys: value = template[key_prefix + key] if key in ['updateMode', 'compression']: iDict[key] = value elif key in ['xstep', 'ystep']: iDict[key] = int(value) elif key in ['unwFile']: iDict['stackDir'] = os.path.dirname(value) elif value: iDict[key] = str(value) return inps
def read_template_file2inps(template_file, inps=None): """Read seed/reference info from template file and update input namespace""" if not inps: inps = cmd_line_parse(['']) inps_dict = vars(inps) template = readfile.read_template(template_file) template = ut.check_template_auto_value(template) prefix = 'mintpy.reference.' key_list = [i for i in list(inps_dict) if prefix+i in template.keys()] for key in key_list: value = template[prefix+key] if value: if key in ['coherenceFile', 'maskFile']: inps_dict[key] = value elif key == 'minCoherence': inps_dict[key] = float(value) key = prefix+'yx' if key in template.keys(): value = template[key] if value: inps.ref_y, inps.ref_x = [int(i) for i in value.split(',')] key = prefix+'lalo' if key in template.keys(): value = template[key] if value: inps.ref_lat, inps.ref_lon = [float(i) for i in value.split(',')] return inps
def read_subset_template2box(template_file): """Read mintpy.subset.lalo/yx option from template file into box type Return None if not specified. """ tmpl = readfile.read_template(template_file) # subset.lalo -> geo_box try: opts = [ i.strip().replace('[', '').replace(']', '') for i in tmpl['mintpy.subset.lalo'].split(',') ] lat0, lat1 = sorted([float(i.strip()) for i in opts[0].split(':')]) lon0, lon1 = sorted([float(i.strip()) for i in opts[1].split(':')]) geo_box = (lon0, lat1, lon1, lat0) except: geo_box = None # subset.yx -> pix_box try: opts = [ i.strip().replace('[', '').replace(']', '') for i in tmpl['mintpy.subset.yx'].split(',') ] y0, y1 = sorted([int(i.strip()) for i in opts[0].split(':')]) x0, x1 = sorted([int(i.strip()) for i in opts[1].split(':')]) pix_box = (x0, y0, x1, y1) except: pix_box = None return pix_box, geo_box
def check_template_auto_value(templateDict, auto_file='../defaults/smallbaselineApp_auto.cfg' ): """Replace auto value based on the input auto config file.""" # Read default template value and turn yes/no to True/False templateAutoFile = os.path.join(os.path.dirname(__file__), auto_file) templateAutoDict = readfile.read_template(templateAutoFile) # Update auto value of input template dict for key, value in templateDict.items(): if value == 'auto' and key in templateAutoDict.keys(): templateDict[key] = templateAutoDict[key] # Change yes --> True and no --> False specialValues = { 'yes': True, 'True': True, 'no': False, 'False': False, 'none': None, } for key, value in templateDict.items(): if value in specialValues.keys(): templateDict[key] = specialValues[value] return templateDict
def main(argv): parser = build_parser() parseArgs = parser.parse_args() username = parseArgs.user password = parseArgs.password host = parseArgs.host db = parseArgs.db working_dir = parseArgs.folder # make sure we have a final / so the below code doesn't break if working_dir[-1] != "/": working_dir += "/" unavco_name = parseArgs.unavco_name attributes_file = working_dir + "add_Attribute.txt" attributes = readfile.read_template(attributes_file) dbController = InsarDatabaseController(username, password, host, db) dbController.connect() for key in attributes: print("Setting attribute " + key + " to " + attributes[key]) if key == "plotAttributes": dbController.add_plot_attribute(unavco_name, key, attributes[key]) else: dbController.add_attribute(unavco_name, key, attributes[key]) dbController.index_table_on("extra_attributes", "area_id", "area_id_idx") dbController.close()
def read_inps2dict(inps): """Read input Namespace object info into inpsDict It grab the following contents into inpsDict 1. inps & all template files 2. configurations: processor, autoPath, updateMode, compression 3. extra metadata: PLATFORM, PROJECT_NAME, 4. translate autoPath """ # Read input info into inpsDict inpsDict = vars(inps) inpsDict['PLATFORM'] = None # Read template file template = {} for fname in inps.template_file: temp = readfile.read_template(fname) temp = ut.check_template_auto_value(temp) template.update(temp) for key, value in template.items(): inpsDict[key] = value if 'processor' in template.keys(): template['mintpy.load.processor'] = template['processor'] prefix = 'mintpy.load.' key_list = [i.split(prefix)[1] for i in template.keys() if i.startswith(prefix)] for key in key_list: value = template[prefix+key] if key in ['processor', 'autoPath', 'updateMode', 'compression']: inpsDict[key] = template[prefix+key] elif value: inpsDict[prefix+key] = template[prefix+key] print('processor : {}'.format(inpsDict['processor'])) if inpsDict['compression'] == False: inpsDict['compression'] = None # PROJECT_NAME --> PLATFORM if not inpsDict['PROJECT_NAME']: cfile = [i for i in list(inps.template_file) if os.path.basename(i) != 'smallbaselineApp.cfg'] inpsDict['PROJECT_NAME'] = sensor.project_name2sensor_name(cfile)[1] msg = 'SAR platform/sensor : ' sensor_name = sensor.project_name2sensor_name(str(inpsDict['PROJECT_NAME']))[0] if sensor_name: msg += str(sensor_name) inpsDict['PLATFORM'] = str(sensor_name) else: msg += 'unknown from project name "{}"'.format(inpsDict['PROJECT_NAME']) print(msg) # update file path with auto if inpsDict.get('autoPath', False): print('use auto path defined in mintpy.defaults.auto_path for options in auto') inpsDict = auto_path.get_auto_path(processor=inpsDict['processor'], work_dir=os.path.dirname(inpsDict['outdir']), template=inpsDict) return inpsDict
def read_template2inps(template_file, inps=None): """Read input template options into Namespace inps""" if not inps: inps = cmd_line_parse() inpsDict = vars(inps) print('read options from template file: '+os.path.basename(template_file)) template = readfile.read_template(inps.template_file) template = ut.check_template_auto_value(template) # Update inps if key existed in template file prefix = 'mintpy.network.' keyList = [i for i in list(inpsDict.keys()) if prefix+i in template.keys()] for key in keyList: value = template[prefix+key] if key in ['coherenceBased', 'areaRatioBased', 'keepMinSpanTree']: inpsDict[key] = value elif value: if key in ['minCoherence', 'minAreaRatio', 'tempBaseMax', 'perpBaseMax']: inpsDict[key] = float(value) elif key in ['connNumMax']: inpsDict[key] = int(value) elif key in ['maskFile', 'referenceFile']: inpsDict[key] = value elif key == 'aoiYX': tmp = [i.replace('[','').replace(']','').strip() for i in value.split(',')] sub_y = sorted([int(i.strip()) for i in tmp[0].split(':')]) sub_x = sorted([int(i.strip()) for i in tmp[1].split(':')]) inps.aoi_pix_box = (sub_x[0], sub_y[0], sub_x[1], sub_y[1]) elif key == 'aoiLALO': tmp = [i.replace('[','').replace(']','').strip() for i in value.split(',')] sub_lat = sorted([float(i.strip()) for i in tmp[0].split(':')]) sub_lon = sorted([float(i.strip()) for i in tmp[1].split(':')]) inps.aoi_geo_box = (sub_lon[0], sub_lat[1], sub_lon[1], sub_lat[0]) # Check lookup file if not inps.lookupFile: print('Warning: no lookup table file found! Can not use '+key+' option without it.') print('skip this option.') inps.aoi_pix_box = None elif key in ['startDate', 'endDate']: inpsDict[key] = ptime.yyyymmdd(value) elif key == 'excludeDate': value = value.replace('[','').replace(']','').replace(',', ' ') inpsDict[key] = ptime.yyyymmdd(value.split()) elif key == 'excludeIfgIndex': value = value.replace('[','').replace(']','').replace(',', ' ') inpsDict[key] += value.split() inpsDict[key] = read_input_index_list(inpsDict[key], stackFile=inps.file) # Turn reset on if 1) no input options found to drop ifgram AND 2) there is template input if all(not i for i in [inps.referenceFile, inps.tempBaseMax, inps.perpBaseMax, inps.connNumMax, inps.excludeIfgIndex, inps.excludeDate, inps.coherenceBased, inps.areaRatioBased, inps.startDate, inps.endDate, inps.reset, inps.manual]): print('No input option found to remove interferogram') print('Keep all interferograms by enable --reset option') inps.reset = True return inps
def read_inps2dict(inps): """Read input Namespace object info into inpsDict""" # Read input info into inpsDict inpsDict = vars(inps) inpsDict['PLATFORM'] = None # Read template file template = {} for fname in inps.template_file: temp = readfile.read_template(fname) temp = ut.check_template_auto_value(temp) template.update(temp) for key, value in template.items(): inpsDict[key] = value if 'processor' in template.keys(): template['mintpy.load.processor'] = template['processor'] prefix = 'mintpy.load.' key_list = [ i.split(prefix)[1] for i in template.keys() if i.startswith(prefix) ] for key in key_list: value = template[prefix + key] if key in ['processor', 'updateMode', 'compression']: inpsDict[key] = template[prefix + key] elif value: inpsDict[prefix + key] = template[prefix + key] if inpsDict['compression'] == False: inpsDict['compression'] = None # PROJECT_NAME --> PLATFORM if not inpsDict['PROJECT_NAME']: cfile = [ i for i in list(inps.template_file) if os.path.basename(i) != 'smallbaselineApp.cfg' ] inpsDict['PROJECT_NAME'] = sensor.project_name2sensor_name(cfile)[1] inpsDict['PLATFORM'] = str( sensor.project_name2sensor_name(str(inpsDict['PROJECT_NAME']))[0]) if inpsDict['PLATFORM']: print('SAR platform/sensor : {}'.format(inpsDict['PLATFORM'])) print('processor: {}'.format(inpsDict['processor'])) # Here to insert code to check default file path for miami user if (auto_path.autoPath and 'SCRATCHDIR' in os.environ and inpsDict['PROJECT_NAME'] is not None and inpsDict['mintpy.load.unwFile']) == 'auto': print(('check auto path setting for Univ of Miami users' ' for processor: {}'.format(inpsDict['processor']))) inpsDict = auto_path.get_auto_path( processor=inpsDict['processor'], project_name=inpsDict['PROJECT_NAME'], template=inpsDict) return inpsDict
def read_template2inps(templateFile, inps=None): """Update inps with options from templateFile""" if not inps: inps = cmd_line_parse() template = readfile.read_template(templateFile) template = ut.check_template_auto_value(template) key = 'mintpy.reference.date' if key in template.keys() and template[key]: inps.refDate = template[key] return inps
def read_inps2dict(inps): """Read input Namespace object info into inpsDict""" # Read input info into inpsDict inpsDict = vars(inps) inpsDict['PLATFORM'] = None # Read template file template = {} for fname in inps.template_file: temp = readfile.read_template(fname) temp = ut.check_template_auto_value(temp) template.update(temp) for key, value in template.items(): inpsDict[key] = value if 'processor' in template.keys(): template['mintpy.load.processor'] = template['processor'] prefix = 'mintpy.load.' key_list = [ i.split(prefix)[1] for i in template.keys() if i.startswith(prefix) ] for key in key_list: value = template[prefix + key] if key in ['processor', 'autoPath', 'updateMode', 'compression']: inpsDict[key] = template[prefix + key] elif value: inpsDict[prefix + key] = template[prefix + key] if inpsDict['compression'] == False: inpsDict['compression'] = None # PROJECT_NAME --> PLATFORM if not inpsDict['PROJECT_NAME']: cfile = [ i for i in list(inps.template_file) if os.path.basename(i) != 'smallbaselineApp.cfg' ] inpsDict['PROJECT_NAME'] = sensor.project_name2sensor_name(cfile)[1] inpsDict['PLATFORM'] = str( sensor.project_name2sensor_name(str(inpsDict['PROJECT_NAME']))[0]) print('SAR platform/sensor : {}'.format(inpsDict['PLATFORM'])) print('processor: {}'.format(inpsDict['processor'])) # update file path with auto if inpsDict['autoPath']: print( 'use auto path defined in mintpy.defaults.auto_path for options in auto' ) inpsDict = auto_path.get_auto_path(processor=inpsDict['processor'], work_dir=os.path.dirname( inpsDict['outdir']), template=inpsDict) return inpsDict
def read_template2inps(templateFile, inps=None): """Update inps with options from templateFile""" if not inps: inps = cmd_line_parse() template = readfile.read_template(templateFile) template = ut.check_template_auto_value(template) key = 'mintpy.reference.date' if key in template.keys() and template[key]: inps.refDate = template[key] return inps
def get_the_latest_default_template_file(work_dir): """Get the latest version of default template file. If an obsolete file exists in the working directory, the existing option values are kept. """ lfile = os.path.join(os.path.dirname(__file__), 'defaults/smallbaselineApp.cfg') #latest version cfile = os.path.join(work_dir, 'smallbaselineApp.cfg') #current version if not os.path.isfile(cfile): print('copy default template file {} to work directory'.format(lfile)) shutil.copy2(lfile, work_dir) else: #cfile is obsolete if any key is missing ldict = readfile.read_template(lfile) cdict = readfile.read_template(cfile) if any([key not in cdict.keys() for key in ldict.keys()]): print('obsolete default template detected, update to the latest version.') shutil.copy2(lfile, work_dir) #keep the existing option value from obsolete template file ut.update_template_file(cfile, cdict) return cfile
def read_template2inps(template_file, inps=None): """Read input template options into Namespace inps""" if not inps: inps = cmd_line_parse() inpsDict = vars(inps) print('read options from template file: '+os.path.basename(template_file)) template = readfile.read_template(inps.template_file) template = ut.check_template_auto_value(template) # Update inps if key existed in template file prefix = 'mintpy.network.' keyList = [i for i in list(inpsDict.keys()) if prefix+i in template.keys()] for key in keyList: value = template[prefix+key] if key in ['coherenceBased', 'keepMinSpanTree']: inpsDict[key] = value elif value: if key in ['minCoherence', 'tempBaseMax', 'perpBaseMax']: inpsDict[key] = float(value) elif key in ['connNumMax']: inpsDict[key] = int(value) elif key in ['maskFile', 'referenceFile']: inpsDict[key] = value elif key == 'aoiYX': tmp = [i.strip() for i in value.split(',')] sub_y = sorted([int(i.strip()) for i in tmp[0].split(':')]) sub_x = sorted([int(i.strip()) for i in tmp[1].split(':')]) inps.aoi_pix_box = (sub_x[0], sub_y[0], sub_x[1], sub_y[1]) elif key == 'aoiLALO': tmp = [i.strip() for i in value.split(',')] sub_lat = sorted([float(i.strip()) for i in tmp[0].split(':')]) sub_lon = sorted([float(i.strip()) for i in tmp[1].split(':')]) inps.aoi_geo_box = (sub_lon[0], sub_lat[1], sub_lon[1], sub_lat[0]) # Check lookup file if not inps.lookupFile: print('Warning: no lookup table file found! Can not use '+key+' option without it.') print('skip this option.') inps.aoi_pix_box = None elif key in ['startDate', 'endDate']: inpsDict[key] = ptime.yyyymmdd(value) elif key == 'excludeDate': inpsDict[key] = ptime.yyyymmdd(value.replace(',', ' ').split()) elif key == 'excludeIfgIndex': inpsDict[key] += value.replace(',', ' ').split() inps.excludeIfgIndex = read_input_index_list(inps.excludeIfgIndex, stackFile=inps.file) # Turn reset on if 1) no input options found to drop ifgram AND 2) there is template input if all(not i for i in [inps.referenceFile, inps.tempBaseMax, inps.perpBaseMax, inps.connNumMax, inps.excludeIfgIndex, inps.excludeDate, inps.coherenceBased, inps.startDate, inps.endDate, inps.reset, inps.manual]): print('No input option found to remove interferogram') print('Keep all interferograms by enable --reset option') inps.reset = True return inps
def read_stripmap_baseline(baseline_file): """Read baseline file generated by ISCE/stripmapStack processor. Example: baselines/20200111_20200125.txt PERP_BASELINE_BOTTOM 173.97914535263297 PERP_BASELINE_TOP 174.05612879066618 """ fDict = readfile.read_template(baseline_file, delimiter=' ') bperp_top = float(fDict['PERP_BASELINE_TOP']) bperp_bottom = float(fDict['PERP_BASELINE_BOTTOM']) return [bperp_top, bperp_bottom]
def read_inps2dict(inps): print('read options from template file: '+os.path.basename(inps.templateFile)) template = readfile.read_template(inps.templateFile) template = ut.check_template_auto_value(template) iDict = vars(inps) key_prefix = 'isce.' key_list = [i.split(key_prefix)[1] for i in template.keys() if i.startswith(key_prefix)] for key in key_list: iDict[key] = template[key_prefix+key] iDict['sensor'], iDict['projectName'] = sensor.project_name2sensor_name(iDict['templateFile']) return iDict
def read_input_attribute(argv, print_msg=True): atr_new = dict() for i in range(1, len(argv)): if os.path.isfile(argv[i]): atr_tmp = readfile.read_template(argv[i]) atr_new.update(atr_tmp) else: atr_tmp = argv[i].split('=') atr_new[atr_tmp[0].strip()] = atr_tmp[1].strip() if print_msg: print("The following attributes will be added/updated, or removed if new value is 'None':") info.print_attributes(atr_new) return atr_new
def read_inps2dict(inps): """Read input Namespace object info into inpsDict""" # Read input info into inpsDict inpsDict = vars(inps) inpsDict['PLATFORM'] = None # Read template file template = {} for fname in inps.template_file: temp = readfile.read_template(fname) temp = ut.check_template_auto_value(temp) template.update(temp) for key, value in template.items(): inpsDict[key] = value if 'processor' in template.keys(): template['mintpy.load.processor'] = template['processor'] prefix = 'mintpy.load.' key_list = [i.split(prefix)[1] for i in template.keys() if i.startswith(prefix)] for key in key_list: value = template[prefix+key] if key in ['processor', 'updateMode', 'compression']: inpsDict[key] = template[prefix+key] elif value: inpsDict[prefix+key] = template[prefix+key] if inpsDict['compression'] == False: inpsDict['compression'] = None # PROJECT_NAME --> PLATFORM if not inpsDict['PROJECT_NAME']: cfile = [i for i in list(inps.template_file) if os.path.basename(i) != 'smallbaselineApp.cfg'] inpsDict['PROJECT_NAME'] = sensor.project_name2sensor_name(cfile)[1] inpsDict['PLATFORM'] = str(sensor.project_name2sensor_name(str(inpsDict['PROJECT_NAME']))[0]) if inpsDict['PLATFORM']: print('platform : {}'.format(inpsDict['PLATFORM'])) print('processor: {}'.format(inpsDict['processor'])) # Here to insert code to check default file path for miami user if (auto_path.autoPath and 'SCRATCHDIR' in os.environ and inpsDict['PROJECT_NAME'] is not None): print(('check auto path setting for Univ of Miami users' ' for processor: {}'.format(inpsDict['processor']))) inpsDict = auto_path.get_auto_path(processor=inpsDict['processor'], project_name=inpsDict['PROJECT_NAME'], template=inpsDict) return inpsDict
def read_template2inps(template_file, inps=None): """Read input template options into Namespace inps""" if not inps: inps = cmd_line_parse() inpsDict = vars(inps) print('read options from tempalte file: '+os.path.basename(inps.template_file)) template = readfile.read_template(template_file) template = ut.check_template_auto_value(template) key_list = [i for i in list(inpsDict.keys()) if key_prefix+i in template.keys()] for key in key_list: value = template[key_prefix+key] if value: if key in ['waterMaskFile']: inpsDict[key] = value return inps
def read_template2inps(template_file, inps=None): """Read input template options into Namespace inps""" if not inps: inps = cmd_line_parse() inpsDict = vars(inps) print('read options from tempalte file: '+os.path.basename(inps.template_file)) template = readfile.read_template(template_file) template = ut.check_template_auto_value(template) key_list = [i for i in list(inpsDict.keys()) if key_prefix+i in template.keys()] for key in key_list: value = template[key_prefix+key] if value: if key in ['waterMaskFile']: inpsDict[key] = value return inps
def read_template2inps(template_file, inps=None): """Read input template options into Namespace inps""" if not inps: inps = cmd_line_parse() inpsDict = vars(inps) print('read options from template file: '+os.path.basename(template_file)) inps.template = readfile.read_template(inps.template_file) inps.template = ut.check_template_auto_value(inps.template) # Coherence-based network modification prefix = 'mintpy.network.' key = prefix+'maskFile' if key in inps.template.keys(): if inps.template[key]: inps.maskFile = inps.template[key] return inps
def prepare_stack(unw_files, meta, update_mode=True): """Prepare .rsc file for all unwrapped interferogram files.""" num_file = len(unw_files) if num_file == 0: raise FileNotFoundError('NO unwrapped interferogram file found!') # write .rsc file for each interferogram file prog_bar = ptime.progressBar(maxValue=num_file) for i, unw_file in enumerate(unw_files): ifg_dir = os.path.dirname(unw_file) ifg_meta = {} # copy over the common metadata for key, value in meta.items(): ifg_meta[key] = value # update from .grd file ifg_meta.update(readfile.read_gdal_vrt(unw_file)) # add DATE12 prm_files = get_prm_files(ifg_dir) date1, date2 = [os.path.splitext(os.path.basename(i))[0] for i in prm_files] ifg_meta['DATE12'] = '{}-{}'.format(ptime.yymmdd(date1), ptime.yymmdd(date2)) # and P_BASELINE_TOP/BOTTOM_HDR baseline_file = os.path.join(ifg_dir, 'baseline.txt') if os.path.isfile(baseline_file): bDict = readfile.read_template(baseline_file) ifg_meta['P_BASELINE_TOP_HDR'] = bDict['B_perpendicular'] ifg_meta['P_BASELINE_BOTTOM_HDR'] = bDict['B_perpendicular'] else: ifg_meta['P_BASELINE_TOP_HDR'] = '0' ifg_meta['P_BASELINE_BOTTOM_HDR'] = '0' msg = 'WARNING: No baseline file found in: {}. '.format(baseline_file) msg += 'Set P_BASELINE* to 0 and continue.' print(msg) # write .rsc file rsc_file = unw_file+'.rsc' writefile.write_roipac_rsc(ifg_meta, rsc_file, update_mode=update_mode, print_msg=False) prog_bar.update(i+1, suffix='{}_{}'.format(date1, date2)) prog_bar.close() return
def read_template2inps(template_file, inps=None): """Read input template file into inps""" if not inps: inps = cmd_line_parse() iDict = vars(inps) print('read options from template file: {}'.format(os.path.basename(template_file))) template = readfile.read_template(template_file) template = ut.check_template_auto_value(template) key_prefix = 'mintpy.load.' keys = [i for i in list(iDict.keys()) if key_prefix+i in template.keys()] for key in keys: value = template[key_prefix+key] if value: iDict[key] = str(value) return inps
def read_template2inps(template_file, inps): """Read input template options into Namespace inps""" if not inps: inps = cmd_line_parse() iDict = vars(inps) template = readfile.read_template(template_file) template = ut.check_template_auto_value(template) keyList = [ i for i in list(iDict.keys()) if key_prefix + i in template.keys() ] for key in keyList: value = template[key_prefix + key] if key in ['weightFunc', 'maskDataset', 'minNormVelocity']: iDict[key] = value elif value: if key in ['maskThreshold', 'minRedundancy', 'memorySize']: iDict[key] = float(value) elif key in ['residualNorm', 'waterMaskFile']: iDict[key] = value # computing configurations dask_key_prefix = 'mintpy.compute.' keyList = [ i for i in list(iDict.keys()) if dask_key_prefix + i in template.keys() ] for key in keyList: value = template[dask_key_prefix + key] if key in ['cluster', 'config']: iDict[key] = value elif value: if key in ['numWorker']: iDict[key] = str(value) elif key in ['memorySize']: iDict[key] = float(value) # False/None --> 'no' for key in ['weightFunc', 'cluster']: if not iDict[key]: iDict[key] = 'no' return inps, template
def read_template2inps(templateFile, inps): """Update inps with mintpy.residualRms.* option from templateFile""" if not inps: inps = cmd_line_parse() inpsDict = vars(inps) print('read options from template file: '+os.path.basename(templateFile)) template = readfile.read_template(templateFile) template = ut.check_template_auto_value(template) prefix = 'mintpy.residualRms.' keyList = [i for i in list(inpsDict.keys()) if prefix+i in template.keys()] for key in keyList: value = template[prefix+key] if value: if key in ['maskFile', 'ramp']: inpsDict[key] = value elif key in ['cutoff']: inpsDict[key] = float(value) return inps
def read_template2inps(templateFile, inps): """Update inps with mintpy.residualRMS.* option from templateFile""" if not inps: inps = cmd_line_parse() inpsDict = vars(inps) print('read options from template file: '+os.path.basename(templateFile)) template = readfile.read_template(templateFile) template = ut.check_template_auto_value(template) prefix = 'mintpy.residualRMS.' keyList = [i for i in list(inpsDict.keys()) if prefix+i in template.keys()] for key in keyList: value = template[prefix+key] if value: if key in ['maskFile', 'deramp']: inpsDict[key] = value elif key in ['cutoff']: inpsDict[key] = float(value) return inps
def read_template2inps(template_file, inps=None): """Read input template file into inps.excludeDate""" if not inps: inps = cmd_line_parse() iDict = vars(inps) print('read options from template file: ' + os.path.basename(template_file)) template = readfile.read_template(template_file) template = ut.check_template_auto_value(template) # Read template option keyList = [ i for i in list(iDict.keys()) if key_prefix + i in template.keys() ] for key in keyList: value = template[key_prefix + key] if key in ['phaseVelocity']: iDict[key] = value elif value: if key in ['polyOrder']: iDict[key] = int(value) elif key in ['excludeDate', 'stepFuncDate']: value = value.replace('[', '').replace(']', '').replace(',', ' ') iDict[key] = ptime.yyyymmdd(value.split()) # computing configurations dask_key_prefix = 'mintpy.compute.' keyList = [ i for i in list(iDict.keys()) if dask_key_prefix + i in template.keys() ] for key in keyList: value = template[dask_key_prefix + key] if key in ['cluster', 'config']: iDict[key] = value elif value: if key in ['numWorker']: iDict[key] = str(value) elif key in ['maxMemory']: iDict[key] = float(value) return inps
def read_template2inps(template_file, inps=None): """Read input template options into Namespace inps""" if not inps: inps = cmd_line_parse() print('read options from template file: '+os.path.basename(template_file)) template = readfile.read_template(template_file) # Coherence-based network modification prefix = 'mintpy.save.hdfEos5.' key = prefix+'update' if key in template.keys() and template[key] == 'yes': inps.update = True key = prefix+'subset' if key in template.keys() and template[key] == 'yes': inps.subset = True return inps
def read_subset_template2box(template_file): """Read mintpy.subset.lalo/yx option from template file into box type Return None if not specified. """ tmpl = readfile.read_template(template_file) try: sub = [i.strip() for i in tmpl['mintpy.subset.lalo'].split(',')] sub_lat = sorted([float(i.strip()) for i in sub[0].split(':')]) sub_lon = sorted([float(i.strip()) for i in sub[1].split(':')]) geo_box = (sub_lon[0], sub_lat[1], sub_lon[1], sub_lat[0]) except: geo_box = None try: sub = [i.strip() for i in tmpl['mintpy.subset.yx'].split(',')] sub_y = sorted([int(i.strip()) for i in sub[0].split(':')]) sub_x = sorted([int(i.strip()) for i in sub[1].split(':')]) pix_box = (sub_x[0], sub_y[0], sub_x[1], sub_y[1]) except: pix_box = None return pix_box, geo_box
def read_template2inps(template_file, inps=None): """Read input template options into Namespace inps""" if not inps: inps = cmd_line_parse() inpsDict = vars(inps) print('read options from template file: '+os.path.basename(template_file)) template = readfile.read_template(inps.template_file) template = ut.check_template_auto_value(template) # Coherence-based network modification prefix = 'mintpy.network.' key = prefix+'maskFile' if key in template.keys(): if template[key]: inps.maskFile = template[key] key = prefix+'minCoherence' if key in template.keys(): if template[key]: inps.coh_thres = float(template[key]) return inps
def check_template_auto_value(templateDict, auto_file='../defaults/smallbaselineApp_auto.cfg'): """Replace auto value based on the input auto config file.""" # Read default template value and turn yes/no to True/False templateAutoFile = os.path.join(os.path.dirname(__file__), auto_file) templateAutoDict = readfile.read_template(templateAutoFile) # Update auto value of input template dict for key, value in templateDict.items(): if value == 'auto' and key in templateAutoDict.keys(): templateDict[key] = templateAutoDict[key] # Change yes --> True and no --> False specialValues = {'yes': True, 'True': True, 'no': False, 'False': False, 'none': None, } for key, value in templateDict.items(): if value in specialValues.keys(): templateDict[key] = specialValues[value] return templateDict
def read_stripmap_baseline(baseline_file): fDict = readfile.read_template(baseline_file, delimiter=' ') bperp_top = float(fDict['PERP_BASELINE_TOP']) bperp_bottom = float(fDict['PERP_BASELINE_BOTTOM']) return [bperp_top, bperp_bottom]
def _read_template(self): # read custom template, to: # 1) update default template # 2) add metadata to ifgramStack file and HDF-EOS5 file self.customTemplate = None if self.customTemplateFile: cfile = self.customTemplateFile # Copy custom template file to inputs directory for backup inputs_dir = os.path.join(self.workDir, 'inputs') if not os.path.isdir(inputs_dir): os.makedirs(inputs_dir) print('create directory:', inputs_dir) if ut.run_or_skip(out_file=os.path.join(inputs_dir, os.path.basename(cfile)), in_file=cfile, check_readable=False) == 'run': shutil.copy2(cfile, inputs_dir) print('copy {} to inputs directory for backup.'.format(os.path.basename(cfile))) # Read custom template print('read custom template file:', cfile) cdict = readfile.read_template(cfile) # correct some loose type errors standardValues = {'def':'auto', 'default':'auto', 'y':'yes', 'on':'yes', 'true':'yes', 'n':'no', 'off':'no', 'false':'no' } for key, value in cdict.items(): if value in standardValues.keys(): cdict[key] = standardValues[value] for key in ['mintpy.deramp', 'mintpy.troposphericDelay.method']: if key in cdict.keys(): cdict[key] = cdict[key].lower().replace('-', '_') if 'processor' in cdict.keys(): cdict['mintpy.load.processor'] = cdict['processor'] # these metadata are used in load_data.py only, not needed afterwards # (in order to manually add extra offset when the lookup table is shifted) # (seen in ROI_PAC product sometimes) for key in ['SUBSET_XMIN', 'SUBSET_YMIN']: if key in cdict.keys(): cdict.pop(key) self.customTemplate = dict(cdict) # Update default template file based on custom template print('update default template based on input custom template') self.templateFile = ut.update_template_file(self.templateFile, self.customTemplate) print('read default template file:', self.templateFile) self.template = readfile.read_template(self.templateFile) self.template = ut.check_template_auto_value(self.template) # correct some loose setup conflicts if self.template['mintpy.geocode'] is False: for key in ['mintpy.save.hdfEos5', 'mintpy.save.kmz']: if self.template[key] is True: self.template['mintpy.geocode'] = True print('Turn ON mintpy.geocode in order to run {}.'.format(key)) break return