def extract_metadata(fname): """Read/extract attributes for PySAR from ROI_PAC .unw, .int, .cor file. For each unwrapped interferogram or spatial coherence file, there are 2 .rsc files: basic metadata file and baseline parameter file. e.g. filt_100901-110117-sim_HDR_4rlks_c10.unw filt_100901-110117-sim_HDR_4rlks_c10.unw.rsc 100901-110117_baseline.rsc Inputs: fname : string, ROI_PAC interferogram filename or path, i.e. /KujuT422F650AlosA/filt_100901-110117-sim_HDR_4rlks_c10.unw Outputs: atr : dict, Attributes dictionary """ # 1. Read basic metadata file basic_rsc_file = fname + '.rsc' if not os.path.isfile(basic_rsc_file) and fname.endswith( '_snap_connect.byt'): unw_rsc_file = '{}.unw.rsc'.format(fname.split('_snap_connect.byt')[0]) copyCmd = 'cp {} {}'.format(unw_rsc_file, basic_rsc_file) print(copyCmd) os.system(copyCmd) basic_dict = readfile.read_roipac_rsc(basic_rsc_file) # return if baseline attributes are already there. if 'P_BASELINE_TOP_HDR' in basic_dict.keys(): return basic_rsc_file atr = {} atr['PROCESSOR'] = 'roipac' atr['FILE_TYPE'] = os.path.splitext(fname)[1] # 2. Read baseline metadata file date1, date2 = basic_dict['DATE12'].split('-') baseline_rsc_file = os.path.dirname( fname) + '/' + date1 + '_' + date2 + '_baseline.rsc' baseline_dict = readfile.read_roipac_rsc(baseline_rsc_file) #print('read '+os.path.basename(basic_rsc_file)+' and '+os.path.basename(baseline_rsc_file)) # 3. Merge atr.update(basic_dict) atr.update(baseline_dict) # Write to rsc file basic_rsc_file = fname + '.rsc' try: atr_orig = readfile.read_roipac_rsc(basic_rsc_file) except: atr_orig = None #keyList = [i for i in atr_orig.keys() if i in atr.keys()] if any((i not in atr_orig.keys() or atr_orig[i] != atr[i]) for i in atr.keys()): print('merging {} into {} '.format(os.path.basename(baseline_rsc_file), os.path.basename(basic_rsc_file))) writefile.write_roipac_rsc(atr, out_file=basic_rsc_file) return basic_rsc_file
def extract_metadata4dem_radar(fname): """Read/extract attribute for .hgt_sim file from Gamma to ROI_PAC Input: sim_150911-150922.hgt_sim sim_150911-150922.rdc.dem Search for: sim_150911-150922.diff_par Output: sim_150911-150922.hgt_sim.rsc sim_150911-150922.rdc.dem.rsc """ atr = {} atr['PROCESSOR'] = 'gamma' atr['FILE_TYPE'] = os.path.splitext(fname)[1] # Get basename of file fname_base = os.path.splitext(fname)[0] for i in range(5): fname_base = os.path.splitext(fname_base)[0] par_file = fname_base+'.diff_par' par_dict = readfile.read_gamma_par(par_file) atr.update(par_dict) # Write to .rsc file rsc_file = fname+'.rsc' try: atr_orig = readfile.read_roipac_rsc(rsc_file) except: atr_orig = dict() if not set(atr.items()).issubset(set(atr_orig.items())): atr_out = {**atr_orig, **atr} print('writing >>> '+os.path.basename(rsc_file)) writefile.write_roipac_rsc(atr_out, out_file=rsc_file) return rsc_file
def extract_metadata4dem_geo(fname): """Read/extract attribute for .dem file from Gamma to ROI_PAC For example, it read input file, sim_150911-150922.utm.dem, find its associated par file, sim_150911-150922.utm.dem.par, read it, and convert to ROI_PAC style and write it to an rsc file, sim_150911-150922.utm.dem.rsc """ atr = {} atr['PROCESSOR'] = 'gamma' atr['FILE_TYPE'] = os.path.splitext(fname)[1] atr['Y_UNIT'] = 'degrees' atr['X_UNIT'] = 'degrees' par_file = fname+'.par' par_dict = readfile.read_gamma_par(par_file) atr.update(par_dict) # Write to .rsc file rsc_file = fname+'.rsc' try: atr_orig = readfile.read_roipac_rsc(rsc_file) except: atr_orig = dict() if not set(atr.items()).issubset(set(atr_orig.items())): atr_out = {**atr_orig, **atr} print('writing >>> '+os.path.basename(rsc_file)) writefile.write_roipac_rsc(atr_out, out_file=rsc_file) return rsc_file
def prepare_stack(inputDir, filePattern, metadata, baselineDict): print('prepare .rsc file for ', filePattern) isceFiles = sorted( glob.glob(os.path.join(os.path.abspath(inputDir), '*', filePattern))) if len(isceFiles) == 0: raise FileNotFoundError( 'no file found in pattern: {}'.format(filePattern)) # write .rsc file for each interferogram file for isceFile in isceFiles: # prepare metadata for current file ifg_metadata = readfile.read_attribute(isceFile, meta_ext='.xml') ifg_metadata.update(metadata) dates = os.path.basename(os.path.dirname(isceFile)).split('_') ifg_metadata = readfile.standardize_metadata_isce( ifg_metadata, dates, baselineDict) # write .rsc file rscFile = isceFile + '.rsc' rscDict = dict() if os.path.isfile(rscFile): rscDict = readfile.read_roipac_rsc(rscFile) # update .rsc file only if there are new metadata key/value if not set(ifg_metadata.items()).issubset(set(rscDict.items())): rscDictOut = {**rscDict, **ifg_metadata} print('writing ', rscFile) writefile.write_roipac_rsc(rscDictOut, rscFile) return
def extract_metadata4dem_geo(fname): """Read/extract attribute for .dem file from Gamma to ROI_PAC For example, it read input file, sim_150911-150922.utm.dem, find its associated par file, sim_150911-150922.utm.dem.par, read it, and convert to ROI_PAC style and write it to an rsc file, sim_150911-150922.utm.dem.rsc """ atr = {} atr['PROCESSOR'] = 'gamma' atr['FILE_TYPE'] = os.path.splitext(fname)[1] atr['Y_UNIT'] = 'degrees' atr['X_UNIT'] = 'degrees' par_file = fname + '.par' #print('read '+os.path.basename(par_file)) par_dict = readfile.read_gamma_par(par_file) atr.update(par_dict) # Write to .rsc file rsc_file = fname + '.rsc' try: atr_orig = readfile.read_roipac_rsc(rsc_file) except: atr_orig = None #keyList = [i for i in atr_orig.keys() if i in atr.keys()] if any((i not in atr_orig.keys() or atr_orig[i] != atr[i]) for i in atr.keys()): print('writing >>> ' + os.path.basename(rsc_file)) writefile.write_roipac_rsc(atr, out_file=rsc_file) return rsc_file
def prepare_metadata4giant(fname, meta_files=None): """Extract metadata from xml files for GIAnT time-series file.""" # check xml files if not meta_files: meta_files = auto_xml_file4giant(fname) if not meta_files: raise FileNotFoundError("no xml file found.") # extract metadata from xml files rsc_files = [i for i in meta_files if i.endswith('.rsc')] xml_files = [i for i in meta_files if i.endswith('.xml')] xml_dict = {} for rsc_file in rsc_files: print('reading {}'.format(rsc_file)) rsc_dict = readfile.read_roipac_rsc(rsc_file) for key in ['length', 'LENGTH', 'FILE_LENGTH', 'width', 'WIDTH']: try: rsc_dict.pop(key) except: pass xml_dict.update(rsc_dict) for xml_file in xml_files: print('reading {}'.format(xml_file)) xml_dict.update(read_giant_xml(xml_file)) if not xml_dict: raise ValueError('No metadata found in file: ' + xml_file) # standardize metadata names xml_dict = readfile.standardize_metadata(xml_dict) # project name sensor_name, project_name = sensor.project_name2sensor_name( os.path.abspath(fname)) if sensor_name: xml_dict['PLATFORM'] = sensor_name if project_name: xml_dict['PROJECT_NAME'] = project_name if sensor_name in project_name: tmp = project_name.split(sensor_name)[1][0] if tmp == 'A': xml_dict['ORBIT_DIRECTION'] = 'ASCENDING' else: xml_dict['ORBIT_DIRECTION'] = 'DESCENDING' # update GIAnT HDF5 file fname = ut.add_attribute(fname, xml_dict, print_msg=True) return fname
def extract_isce_metadata(meta_file, geom_dir=None, rsc_file=None, update_mode=True): """Extract metadata from ISCE stack products Parameters: meta_file : str, path of metadata file, master/IW1.xml or masterShelve/data.dat geom_dir : str, path of geometry directory. rsc_file : str, output file name of ROIPAC format rsc file Returns: metadata : dict """ if not rsc_file: rsc_file = os.path.join(os.path.dirname(meta_file), 'data.rsc') # check existing rsc_file if update_mode and ut.run_or_skip(rsc_file, in_file=meta_file, check_readable=False) == 'skip': return readfile.read_roipac_rsc(rsc_file) # 1. extract metadata from XML / shelve file fbase = os.path.basename(meta_file) if fbase.startswith("IW"): print('extract metadata from ISCE/topsStack xml file:', meta_file) metadata = extract_tops_metadata(meta_file) elif fbase.startswith("data"): print('extract metadata from ISCE/stripmapStack shelve file:', meta_file) metadata = extract_stripmap_metadata(meta_file) elif fbase.endswith(".xml"): metadata = extract_stripmap_metadata(meta_file) else: raise ValueError("unrecognized ISCE metadata file: {}".format(meta_file)) # 2. extract metadata from geometry file if geom_dir: metadata = extract_geometry_metadata(geom_dir, metadata) # 3. common metadata metadata['PROCESSOR'] = 'isce' metadata['ANTENNA_SIDE'] = '-1' # convert all value to string format for key, value in metadata.items(): metadata[key] = str(value) # write to .rsc file metadata = readfile.standardize_metadata(metadata) if rsc_file: print('writing ', rsc_file) writefile.write_roipac_rsc(metadata, rsc_file) return metadata
def write_roipac_rsc(metadata, out_file, update_mode=False, print_msg=False): """Write attribute dict into ROI_PAC .rsc file Inputs: metadata : dict, attributes dictionary out_file : rsc file name, to which attribute is writen update_mode : bool, skip writing if 1) output file existed AND 2) no new metadata key/value print_msg : bool, print message Output: out_file """ run = True if update_mode: rsc_dict = dict() if os.path.isfile(out_file): rsc_dict = readfile.read_roipac_rsc(out_file) # update .rsc file only if there are new metadata key/value if set(metadata.items()).issubset(set(rsc_dict.items())): run = False if run: # Convert PYSAR attributes to ROI_PAC attributes if 'LENGTH' in metadata.keys(): metadata['FILE_LENGTH'] = metadata['LENGTH'] # Convert 3.333e-4 to 0.0003333 if 'X_STEP' in metadata.keys(): metadata['X_STEP'] = str(float(metadata['X_STEP'])) metadata['Y_STEP'] = str(float(metadata['Y_STEP'])) metadata['X_FIRST'] = str(float(metadata['X_FIRST'])) metadata['Y_FIRST'] = str(float(metadata['Y_FIRST'])) # writing .rsc file if print_msg: print('write', out_file) maxDigit = max([len(key) for key in metadata.keys()] + [2]) with open(out_file, 'w') as f: for key in sorted(metadata.keys()): f.write('{k:<{d}} {v}\n'.format(k=str(key), d=maxDigit, v=str(metadata[key]))) return out_file
def extract_metadata4geometry_geo(fname): """Read/extract attribute for *.dem / *.UTM_TO_RDC file from Gamma to ROI_PAC Inputs: sim_20070813_20080310.utm.dem sim_20070813_20080310.UTM_TO_RDC Search for: sim_20070813_20080310.utm.dem.par Outputs: sim_20070813_20080310.utm.dem.rsc sim_20070813_20080310.UTM_TO_RDC.rsc """ # Get/read GAMMA par file ext = os.path.splitext(fname)[1] if ext in ['.UTM_TO_RDC']: par_file = os.path.splitext(fname)[0] + '.utm.dem.par' elif fnames[0].endswith('.utm.dem'): par_file = fname + '.par' par_dict = readfile.read_gamma_par(par_file) # initiate ROIPAC dict atr = {} atr['PROCESSOR'] = 'gamma' atr['FILE_TYPE'] = ext atr['Y_UNIT'] = 'degrees' atr['X_UNIT'] = 'degrees' atr.update(par_dict) # Write to .rsc file rsc_file = fname + '.rsc' try: atr_orig = readfile.read_roipac_rsc(rsc_file) except: atr_orig = dict() if not set(atr.items()).issubset(set(atr_orig.items())): atr_out = {**atr_orig, **atr} print('writing >>> ' + os.path.basename(rsc_file)) writefile.write_roipac_rsc(atr_out, out_file=rsc_file) return rsc_file
def main(iargs=None): inps = cmd_line_parse(iargs) baselineDict = {} if inps.baselineDir: baselineDict = baseline_timeseries(inps.baselineDir) metadata = {} if inps.xmlFile: rscFile = os.path.join(os.path.dirname(inps.xmlFile), 'data.rsc') if ut.run_or_skip(rscFile, in_file=inps.xmlFile, check_readable=False) == 'run': metadata = extract_isce_metadata(inps.xmlFile, rscFile) else: metadata = readfile.read_roipac_rsc(rscFile) if inps.geometryDir: metadata = prepare_geometry(inps.geometryDir, metadata) if inps.ifgramDir and inps.ifgramFiles: for namePattern in inps.ifgramFiles: prepare_stack(inps.ifgramDir, namePattern, metadata, baselineDict) print('Done.') return
def extract_metadata4dem_radar(fname): """Read/extract attribute for .hgt_sim file from Gamma to ROI_PAC Input: sim_150911-150922.hgt_sim sim_150911-150922.rdc.dem Search for: sim_150911-150922.diff_par Output: sim_150911-150922.hgt_sim.rsc sim_150911-150922.rdc.dem.rsc """ atr = {} atr['PROCESSOR'] = 'gamma' atr['FILE_TYPE'] = os.path.splitext(fname)[1] # Get basename of file fname_base = os.path.splitext(fname)[0] for i in range(5): fname_base = os.path.splitext(fname_base)[0] par_file = fname_base + '.diff_par' #print('read '+os.path.basename(par_file)) par_dict = readfile.read_gamma_par(par_file) atr.update(par_dict) # Write to .rsc file rsc_file = fname + '.rsc' try: atr_orig = readfile.read_roipac_rsc(rsc_file) except: atr_orig = None #keyList = [i for i in atr_orig.keys() if i in atr.keys()] if any((i not in atr_orig.keys() or atr_orig[i] != atr[i]) for i in atr.keys()): print('writing >>> ' + os.path.basename(rsc_file)) writefile.write_roipac_rsc(atr, out_file=rsc_file) return rsc_file
def extract_metadata4interferogram(fname): """Read/extract attributes for PySAR from Gamma .unw, .cor and .int file Parameters: fname : str, Gamma interferogram filename or path, i.e. /PopoSLT143TsxD/diff_filt_HDR_130118-130129_4rlks.unw Returns: atr : dict, Attributes dictionary """ file_dir = os.path.dirname(fname) file_basename = os.path.basename(fname) rsc_file = fname+'.rsc' # if os.path.isfile(rsc_file): # return rsc_file atr = {} atr['PROCESSOR'] = 'gamma' atr['FILE_TYPE'] = os.path.splitext(fname)[1] # Get info: date12, num of loooks try: date12 = str(re.findall('\d{8}[-_]\d{8}', file_basename)[0]) except: date12 = str(re.findall('\d{6}[-_]\d{6}', file_basename)[0]) m_date, s_date = date12.replace('-', '_').split('_') atr['DATE12'] = ptime.yymmdd(m_date)+'-'+ptime.yymmdd(s_date) lks = os.path.splitext(file_basename.split(date12)[1])[0] # Read .off and .par file off_file = file_dir+'/*'+date12+lks+'.off' m_par_file = [file_dir+'/*'+m_date+lks+i for i in ['.amp.par', '.ramp.par']] s_par_file = [file_dir+'/*'+s_date+lks+i for i in ['.amp.par', '.ramp.par']] try: off_file = ut.get_file_list(off_file)[0] except: print('\nERROR: Can not find .off file, it supposed to be like: '+off_file) try: m_par_file = ut.get_file_list(m_par_file)[0] except: print('\nERROR: Can not find master date .par file, it supposed to be like: '+m_par_file) try: s_par_file = ut.get_file_list(s_par_file)[0] except: print('\nERROR: Can not find slave date .par file, it supposed to be like: '+s_par_file) par_dict = readfile.read_gamma_par(m_par_file) off_dict = readfile.read_gamma_par(off_file) atr.update(par_dict) atr.update(off_dict) # Perp Baseline Info atr = get_perp_baseline(m_par_file, s_par_file, off_file, atr) # LAT/LON_REF1/2/3/4 atr = get_lalo_ref(m_par_file, atr) # Write to .rsc file try: atr_orig = readfile.read_roipac_rsc(rsc_file) except: atr_orig = dict() if not set(atr.items()).issubset(set(atr_orig.items())): atr_out = {**atr_orig, **atr} print('merge %s, %s and %s into %s' % (os.path.basename(m_par_file), os.path.basename(s_par_file), os.path.basename(off_file), os.path.basename(rsc_file))) writefile.write_roipac_rsc(atr_out, out_file=rsc_file) return rsc_file
def prepare_geometry(geometryDir, metadata=dict()): """Prepare and extract metadata from geometry files""" def get_nonzero_row_number(data, buffer=2): """Find the first and last row number of rows without zero value, for multiple swaths data""" if np.all(data): r0, r1 = 0 + buffer, -1 - buffer else: row_flag = np.sum(data != 0., axis=1) == data.shape[1] row_idx = np.where(row_flag)[0] r0, r1 = row_idx[0] + buffer, row_idx[-1] - buffer return r0, r1 print('prepare .rsc file for geometry files') isceFiles = [ os.path.join(os.path.abspath(geometryDir), '{}.rdr'.format(i)) for i in ['hgt', 'lat', 'lon', 'los', 'shadowMask'] ] isceFiles = [i for i in isceFiles if os.path.isfile(i)] # get A/RLOOKS metadata = extract_multilook_number(metadata, geometryDir) # get LAT/LON_REF1/2/3/4 and HEADING into metadata for isceFile in isceFiles: if 'lat' in os.path.basename(isceFile): data = readfile.read(isceFile)[0] r0, r1 = get_nonzero_row_number(data) metadata['LAT_REF1'] = str(data[r0, 0]) metadata['LAT_REF2'] = str(data[r0, -1]) metadata['LAT_REF3'] = str(data[r1, 0]) metadata['LAT_REF4'] = str(data[r1, -1]) if 'lon' in os.path.basename(isceFile): data = readfile.read(isceFile)[0] r0, r1 = get_nonzero_row_number(data) metadata['LON_REF1'] = str(data[r0, 0]) metadata['LON_REF2'] = str(data[r0, -1]) metadata['LON_REF3'] = str(data[r1, 0]) metadata['LON_REF4'] = str(data[r1, -1]) if 'los' in os.path.basename(isceFile): data = readfile.read(isceFile, datasetName='inc')[0] data[data == 0.] = np.nan az_angle = np.nanmean(data) # convert isce azimuth angle to roipac orbit heading angle head_angle = -1 * (270 + az_angle) head_angle -= np.round(head_angle / 360.) * 360. metadata['HEADING'] = str(head_angle) # write rsc file for each geometry file for isceFile in isceFiles: # prepare metadata for current file geom_metadata = readfile.read_attribute(isceFile, meta_ext='.xml') geom_metadata.update(metadata) geom_metadata = readfile.standardize_metadata_isce(geom_metadata) # write .rsc file rscFile = isceFile + '.rsc' rscDict = dict() if os.path.isfile(rscFile): rscDict = readfile.read_roipac_rsc(rscFile) # update .rsc file only if there are new metadata key/value if not set(geom_metadata.items()).issubset(set(rscDict.items())): rscDictOut = {**rscDict, **geom_metadata} print('writing ', rscFile) writefile.write_roipac_rsc(rscDictOut, rscFile) return metadata
def extract_metadata4geometry_radar(fname): """Read/extract attribute for .hgt_sim file from Gamma to ROI_PAC Input: sim_20070813_20080310.hgt_sim sim_20070813_20080310.rdc.dem Search for: sim_20070813_20080310.diff_par Output: sim_20070813_20080310.hgt_sim.rsc sim_20070813_20080310.rdc.dem.rsc """ # Get/read GAMMA par file # for loop to get rid of multiple dot in filename fname_base = os.path.splitext(fname)[0] for i in range(5): fname_base = os.path.splitext(fname_base)[0] par_file = fname_base + '.diff_par' par_dict = readfile.read_gamma_par(par_file) # Get/read LAT/LON_REF1/2/3/4 msg = 'grab LAT/LON_REF1/2/3/4 from par file: ' # get date of one acquisition try: m_date = str(re.findall('\d{8}', fname_base)[0]) except: m_date = str(re.findall('\d{6}', fname_base)[0]) # search existing par file geom_dir = os.path.dirname(fname) #PROJECT_DIR/geom_master ifg_dir = os.path.join(geom_dir, '../*/{}_*'.format( m_date)) #PROJECT_DIR/interferograms/{m_date}_20141225 m_par_files = [ os.path.join(geom_dir, '*{}*{}'.format(m_date, ext)) for ext in PAR_EXT_LIST ] m_par_files += [ os.path.join(ifg_dir, '*{}*{}'.format(m_date, ext)) for ext in PAR_EXT_LIST ] m_par_files = ut.get_file_list(m_par_files) # read par file if len(m_par_files) > 0: m_par_file = m_par_files[0] msg += m_par_file par_dict = get_lalo_ref(m_par_file, par_dict) else: msg += ' no par file found with date: {}'.format(m_date) print(msg) # initiate ROIPAC dict atr = {} atr['PROCESSOR'] = 'gamma' atr['FILE_TYPE'] = os.path.splitext(fname)[1] atr.update(par_dict) # Write to .rsc file rsc_file = fname + '.rsc' try: atr_orig = readfile.read_roipac_rsc(rsc_file) except: atr_orig = dict() if not set(atr.items()).issubset(set(atr_orig.items())): atr_out = {**atr_orig, **atr} print('writing >>> ' + os.path.basename(rsc_file)) writefile.write_roipac_rsc(atr_out, out_file=rsc_file) return rsc_file