def print_date_list(fname, disp_num=False, drop_ifgram=False, print_msg=False): """Print time/date info of file""" atr = readfile.read_attribute(fname) k = atr['FILE_TYPE'] dateList = None if k in ['timeseries']: dateList = timeseries(fname).get_date_list() elif k == 'HDFEOS': obj = HDFEOS(fname) obj.open(print_msg=False) dateList = obj.dateList elif k == 'giantTimeseries': obj = giantTimeseries(fname) obj.open(print_msg=False) dateList = obj.dateList elif k in ['ifgramStack']: dateList = ifgramStack(fname).get_date12_list(dropIfgram=drop_ifgram) elif k in ['giantIfgramStack']: obj = giantIfgramStack(fname) obj.open(print_msg=False) dateList = obj.date12List else: print('--date option can not be applied to {} file, ignore it.'.format(k)) if print_msg and dateList is not None: for i in range(len(dateList)): if disp_num: print('{}\t{}'.format(dateList[i], i)) else: print(dateList[i]) return dateList
def open(self): atr = readfile.read_attribute(self.insar_file) k = atr['FILE_TYPE'] if k == 'timeseries': ts_obj = timeseries(self.insar_file) elif k == 'giantTimeseries': ts_obj = giantTimeseries(self.insar_file) ts_obj.open(print_msg=False) self.metadata = dict(ts_obj.metadata) self.num_date = ts_obj.numDate self.insar_datetime = ts_obj.times self.read_gps() self.read_insar() self.calculate_rmse() return
def print_date_list(fname, disp_ifgram='all', disp_num=False, print_msg=False): """Print time/date info of file""" k = readfile.read_attribute(fname)['FILE_TYPE'] dateList = None if k in ['timeseries']: dateList = timeseries(fname).get_date_list() elif k == 'HDFEOS': dateList = HDFEOS(fname).get_date_list() elif k == 'giantTimeseries': dateList = giantTimeseries(fname).get_date_list() elif k in ['giantIfgramStack']: dateList = giantIfgramStack(fname).get_date12_list() elif k in ['ifgramStack']: obj = ifgramStack(fname) obj.open(print_msg=False) dateListAll = obj.get_date12_list(dropIfgram=False) dateListKept = obj.get_date12_list(dropIfgram=True) # show dropped ifgram or not if disp_ifgram == 'all': dateList = list(dateListAll) elif disp_ifgram == 'kept': dateList = list(dateListKept) else: dateList = sorted(list(set(dateListAll) - set(dateListKept))) else: print('--date option can not be applied to {} file, ignore it.'.format( k)) # print list info if print_msg and dateList is not None: for d in dateList: if disp_num: if k in ['ifgramStack']: num = dateListAll.index(d) else: num = dateList.index(d) msg = '{}\t{}'.format(d, num) else: msg = d print(msg) return dateList
def read_date_info(inps): """Get inps.excludeDate full list Inputs: inps - Namespace, Output: inps.excludeDate - list of string for exclude date in YYYYMMDD format """ if inps.key == 'timeseries': tsobj = timeseries(inps.timeseries_file) elif inps.key == 'giantTimeseries': tsobj = giantTimeseries(inps.timeseries_file) elif inps.key == 'HDFEOS': tsobj = HDFEOS(inps.timeseries_file) tsobj.open() inps.excludeDate = read_exclude_date(inps, tsobj.dateList) # Date used for estimation inps.dateList inps.dateList = [i for i in tsobj.dateList if i not in inps.excludeDate] inps.numDate = len(inps.dateList) print('-' * 50) print('dates from input file: {}\n{}'.format(tsobj.numDate, tsobj.dateList)) print('-' * 50) if len(inps.dateList) == len(tsobj.dateList): print('using all dates to calculate the velocity') else: print('dates used to estimate the velocity: {}\n{}'.format( inps.numDate, inps.dateList)) print('-' * 50) # flag array for ts data reading inps.dropDate = np.array( [i not in inps.excludeDate for i in tsobj.dateList], dtype=np.bool_) # output file name if not inps.outfile: outname = 'velocity' if inps.key == 'giantTimeseries': prefix = os.path.basename(inps.timeseries_file).split('PARAMS')[0] outname = prefix + outname outname += '.h5' inps.outfile = outname return inps
def read_attribute(fname, datasetName=None, standardize=True, meta_ext=None): """Read attributes of input file into a dictionary Parameters: fname : str, path/name of data file datasetName : str, name of dataset of interest, for file with multiple datasets e.g. unwrapPhase in ifgramStack.h5 coherence in ifgramStack.h5 height in geometryRadar.h5 latitude in geometryRadar.h5 ... standardize : bool, grab standardized metadata key name Returns: atr : dict, attributes dictionary """ fbase, fext = os.path.splitext(os.path.basename(fname)) fext = fext.lower() if not os.path.isfile(fname): msg = 'input file not existed: {}\n'.format(fname) msg += 'current directory: ' + os.getcwd() raise Exception(msg) # HDF5 files if fext in ['.h5', '.he5']: f = h5py.File(fname, 'r') g1_list = [i for i in f.keys() if isinstance(f[i], h5py.Group)] d1_list = [ i for i in f.keys() if isinstance(f[i], h5py.Dataset) and f[i].ndim >= 2 ] # FILE_TYPE - k if any(i in d1_list for i in ['unwrapPhase']): k = 'ifgramStack' elif any(i in d1_list for i in ['height', 'latitude', 'azimuthCoord']): k = 'geometry' elif any(i in g1_list + d1_list for i in ['timeseries', 'displacement']): k = 'timeseries' elif 'HDFEOS' in g1_list: k = 'HDFEOS' elif 'recons' in d1_list: k = 'giantTimeseries' elif any(i in d1_list for i in ['igram', 'figram']): k = 'giantIfgramStack' elif any(i in g1_list for i in multi_group_hdf5_file): # old pysar format k = list(set(g1_list) & set(multi_group_hdf5_file))[0] elif len(d1_list) > 0: k = d1_list[0] elif len(g1_list) > 0: k = g1_list[0] else: raise ValueError('unrecognized file type: ' + fname) # metadata dict if k == 'giantTimeseries': atr = giantTimeseries(fname).get_metadata() elif k == 'giantIfgramStack': atr = giantIfgramStack(fname).get_metadata() else: if len(f.attrs) > 0 and 'WIDTH' in f.attrs.keys(): atr = dict(f.attrs) else: # grab the list of attrs in HDF5 file global atr_list def get_hdf5_attrs(name, obj): global atr_list if len(obj.attrs) > 0 and 'WIDTH' in obj.attrs.keys(): atr_list.append(dict(obj.attrs)) atr_list = [] f.visititems(get_hdf5_attrs) # use the attrs with most items if atr_list: num_list = [len(i) for i in atr_list] atr = atr_list[np.argmax(num_list)] else: raise ValueError('No attribute WIDTH found in file:', fname) # decode string format for key, value in atr.items(): try: atr[key] = value.decode('utf8') except: atr[key] = value # attribute identified by PySAR # 1. FILE_TYPE atr['FILE_TYPE'] = str(k) # 2. DATA_TYPE ds = None if datasetName and datasetName in f.keys(): ds = f[datasetName] else: # get the 1st dataset global ds_list def get_hdf5_dataset(name, obj): global ds_list if isinstance(obj, h5py.Dataset) and obj.ndim >= 2: ds_list.append(obj) ds_list = [] f.visititems(get_hdf5_dataset) if ds_list: ds = ds_list[0] if ds is not None: atr['DATA_TYPE'] = str(ds.dtype) f.close() # 3. PROCESSOR if 'INSAR_PROCESSOR' in atr.keys(): atr['PROCESSOR'] = atr['INSAR_PROCESSOR'] if 'PROCESSOR' not in atr.keys(): atr['PROCESSOR'] = 'pysar' else: # get existing metadata files metafile_exts = ['.rsc', '.xml', '.aux.xml', '.par', '.hdr'] if meta_ext: metafile_exts = [i for i in metafile_exts if i.endswith(meta_ext)] metafile_exts = [i for i in metafile_exts if os.path.isfile(fname + i)] if len(metafile_exts) == 0: raise FileNotFoundError( 'No metadata file found for data file: {}'.format(fname)) # Read metadata file and FILE_TYPE while fext in ['.geo', '.rdr']: fbase, fext = os.path.splitext(fbase) if not fext: fext = fbase metafile0 = fname + metafile_exts[0] if metafile0.endswith('.rsc'): atr = read_roipac_rsc(metafile0) if 'FILE_TYPE' not in atr.keys(): atr['FILE_TYPE'] = fext elif metafile0.endswith('.xml'): atr = read_isce_xml(metafile0) if 'FILE_TYPE' not in atr.keys(): atr['FILE_TYPE'] = atr.get('image_type', fext) elif metafile0.endswith('.par'): atr = read_gamma_par(metafile0) atr['FILE_TYPE'] = fext elif metafile0.endswith('.hdr'): atr = read_template(metafile0) atr['DATA_TYPE'] = ENVI2NUMPY_DATATYPE[xmlDict.get( 'data type', '4')] atr['FILE_TYPE'] = atr['file type'] # PROCESSOR if any(i.endswith(('.xml', '.hdr')) for i in metafile_exts): atr['PROCESSOR'] = 'isce' #atr.update(read_isce_xml(fname+'.xml')) elif any(i.endswith('.par') for i in metafile_exts): atr['PROCESSOR'] = 'gamma' elif any(i.endswith('.rsc') for i in metafile_exts): if 'PROCESSOR' not in atr.keys(): atr['PROCESSOR'] = 'roipac' if 'PROCESSOR' not in atr.keys(): atr['PROCESSOR'] = 'pysar' # UNIT k = atr['FILE_TYPE'].replace('.', '') if k == 'ifgramStack': if datasetName and datasetName in datasetUnitDict.keys(): atr['UNIT'] = datasetUnitDict[datasetName] else: atr['UNIT'] = 'radian' elif 'UNIT' not in atr.keys(): if datasetName and datasetName in datasetUnitDict.keys(): atr['UNIT'] = datasetUnitDict[datasetName] elif k in datasetUnitDict.keys(): atr['UNIT'] = datasetUnitDict[k] else: atr['UNIT'] = '1' # FILE_PATH atr['FILE_PATH'] = os.path.abspath(fname) if standardize: atr = standardize_metadata(atr) return atr
def get_slice_list(fname): """Get list of 2D slice existed in file (for display)""" fbase, fext = os.path.splitext(os.path.basename(fname)) fext = fext.lower() atr = read_attribute(fname) k = atr['FILE_TYPE'] global slice_list # HDF5 Files if fext in ['.h5', '.he5']: with h5py.File(fname, 'r') as f: d1_list = [i for i in f.keys() if isinstance(f[i], h5py.Dataset)] if k == 'timeseries' and k in d1_list: obj = timeseries(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['geometry'] and k not in d1_list: obj = geometry(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['ifgramStack']: obj = ifgramStack(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['HDFEOS']: obj = HDFEOS(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['giantTimeseries']: obj = giantTimeseries(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['giantIfgramStack']: obj = giantIfgramStack(fname) obj.open(print_msg=False) slice_list = obj.sliceList else: ## Find slice by walking through the file structure length, width = int(atr['LENGTH']), int(atr['WIDTH']) def get_hdf5_2d_dataset(name, obj): global slice_list if isinstance(obj, h5py.Dataset) and obj.shape[-2:] == (length, width): if obj.ndim == 2: slice_list.append(name) else: warnings.warn( 'file has un-defined {}D dataset: {}'.format( obj.ndim, name)) slice_list = [] with h5py.File(fname, 'r') as f: f.visititems(get_hdf5_2d_dataset) # Binary Files else: if fext.lower() in ['.trans', '.utm_to_rdc']: slice_list = ['rangeCoord', 'azimuthCoord'] elif fbase.startswith('los'): slice_list = ['incidenceAngle', 'azimuthAngle'] elif atr.get('number_bands', '1') == '2' and 'unw' not in k: slice_list = ['band1', 'band2'] else: slice_list = [''] return slice_list
def read_init_info(inps): # Time Series Info ts_file0 = inps.timeseries_file[0] atr = readfile.read_attribute(ts_file0) inps.key = atr['FILE_TYPE'] if inps.key == 'timeseries': obj = timeseries(ts_file0) elif inps.key == 'giantTimeseries': obj = giantTimeseries(ts_file0) elif inps.key == 'HDFEOS': obj = HDFEOS(ts_file0) else: raise ValueError('input file is {}, not timeseries.'.format(inps.key)) obj.open() if not inps.file_label: inps.file_label = [ str(i) for i in list(range(len(inps.timeseries_file))) ] # default mask file if not inps.mask_file and 'masked' not in ts_file0: dir_name = os.path.dirname(ts_file0) if 'Y_FIRST' in atr.keys(): inps.mask_file = os.path.join(dir_name, 'geo_maskTempCoh.h5') else: inps.mask_file = os.path.join(dir_name, 'maskTempCoh.h5') if not os.path.isfile(inps.mask_file): inps.mask_file = None # date info inps.date_list = obj.dateList if inps.start_date: inps.date_list = [ i for i in inps.date_list if int(i) >= int(inps.start_date) ] if inps.end_date: inps.date_list = [ i for i in inps.date_list if int(i) <= int(inps.end_date) ] inps.num_date = len(inps.date_list) inps.dates, inps.yearList = ptime.date_list2vector(inps.date_list) (inps.ex_date_list, inps.ex_dates, inps.ex_flag) = read_exclude_date(inps.ex_date_list, inps.date_list) # initial display index if obj.metadata['REF_DATE'] in inps.date_list: inps.ref_idx = inps.date_list.index(obj.metadata['REF_DATE']) else: inps.ref_idx = 0 if inps.ref_date: inps.ref_idx = inps.date_list.index(inps.ref_date) if not inps.init_idx: if inps.ref_idx < inps.num_date / 2.: inps.init_idx = -3 else: inps.init_idx = 3 # Display Unit (inps.disp_unit, inps.unit_fac) = pp.scale_data2disp_unit(metadata=atr, disp_unit=inps.disp_unit)[1:3] # Read Error List inps.error_ts = None inps.ex_error_ts = None if inps.error_file: error_fileContent = np.loadtxt(inps.error_file, dtype=bytes).astype(str) inps.error_ts = error_fileContent[:, 1].astype( np.float) * inps.unit_fac if inps.ex_date_list: e_ts = inps.error_ts[:] inps.ex_error_ts = e_ts[inps.ex_flag == 0] inps.error_ts = e_ts[inps.ex_flag == 1] # Zero displacement for 1st acquisition if inps.zero_first: inps.zero_idx = min(0, np.min(np.where(inps.ex_flag)[0])) # default lookup table file if not inps.lookup_file: inps.lookup_file = ut.get_lookup_file('./INPUTS/geometryRadar.h5') inps.coord = ut.coordinate(atr, inps.lookup_file) # size and lalo info inps.pix_box, inps.geo_box = subset.subset_input_dict2box(vars(inps), atr) inps.pix_box = inps.coord.check_box_within_data_coverage(inps.pix_box) inps.geo_box = inps.coord.box_pixel2geo(inps.pix_box) # Out message data_box = (0, 0, obj.width, obj.length) print('data coverage in y/x: ' + str(data_box)) print('subset coverage in y/x: ' + str(inps.pix_box)) print('data coverage in lat/lon: ' + str(inps.coord.box_pixel2geo(data_box))) print('subset coverage in lat/lon: ' + str(inps.geo_box)) print( '------------------------------------------------------------------------' ) # reference pixel if not inps.ref_lalo and 'REF_LAT' in atr.keys(): inps.ref_lalo = (float(atr['REF_LAT']), float(atr['REF_LON'])) if inps.ref_lalo: if inps.ref_lalo[1] > 180.: inps.ref_lalo[1] -= 360. inps.ref_yx = inps.coord.geo2radar(inps.ref_lalo[0], inps.ref_lalo[1], print_msg=False)[0:2] if not inps.ref_yx: inps.ref_yx = [int(atr['REF_Y']), int(atr['REF_X'])] # Initial Pixel Coord if inps.lalo: inps.yx = inps.coord.geo2radar(inps.lalo[0], inps.lalo[1], print_msg=False)[0:2] try: inps.lalo = inps.coord.radar2geo(inps.yx[0], inps.yx[1], print_msg=False)[0:2] except: inps.lalo = None # Flip up-down / left-right if inps.auto_flip: inps.flip_lr, inps.flip_ud = pp.auto_flip_direction(atr) # display unit ans wrap # if wrap_step == 2*np.pi (default value), set disp_unit_v = radian; # otherwise set disp_unit_v = disp_unit inps.disp_unit_v = inps.disp_unit if inps.wrap: inps.range2phase = -4. * np.pi / float(atr['WAVELENGTH']) if 'cm' == inps.disp_unit.split('/')[0]: inps.range2phase /= 100. elif 'mm' == inps.disp_unit.split('/')[0]: inps.range2phase /= 1000. elif 'm' == inps.disp_unit.split('/')[0]: inps.range2phase /= 1. else: raise ValueError('un-recognized display unit: {}'.format( inps.disp_unit)) if (inps.wrap_range[1] - inps.wrap_range[0]) == 2 * np.pi: inps.disp_unit_v = 'radian' inps.vlim = inps.wrap_range inps.cbar_label = 'Displacement [{}]'.format(inps.disp_unit_v) return inps, atr
def diff_file(file1, file2, outFile=None, force=False): """Subtraction/difference of two input files""" if not outFile: fbase, fext = os.path.splitext(file1) if len(file2) > 1: raise ValueError('Output file name is needed for more than 2 files input.') outFile = '{}_diff_{}{}'.format(fbase, os.path.splitext(os.path.basename(file2[0]))[0], fext) print('{} - {} --> {}'.format(file1, file2, outFile)) # Read basic info atr1 = readfile.read_attribute(file1) k1 = atr1['FILE_TYPE'] atr2 = readfile.read_attribute(file2[0]) k2 = atr2['FILE_TYPE'] print('input files are: {} and {}'.format(k1, k2)) if k1 == 'timeseries': if k2 not in ['timeseries', 'giantTimeseries']: raise Exception('Input multiple dataset files are not the same file type!') if len(file2) > 1: raise Exception(('Only 2 files substraction is supported for time-series file,' ' {} input.'.format(len(file2)+1))) obj1 = timeseries(file1) obj1.open() if k2 == 'timeseries': obj2 = timeseries(file2[0]) unit_fac = 1. elif k2 == 'giantTimeseries': obj2 = giantTimeseries(file2[0]) unit_fac = 0.001 obj2.open() ref_date, ref_y, ref_x = _check_reference(obj1.metadata, obj2.metadata) # check dates shared by two timeseries files dateListShared = [i for i in obj1.dateList if i in obj2.dateList] dateShared = np.ones((obj1.numDate), dtype=np.bool_) if dateListShared != obj1.dateList: print('WARNING: {} does not contain all dates in {}'.format(file2, file1)) if force: dateExcluded = list(set(obj1.dateList) - set(dateListShared)) print('Continue and enforce the differencing for their shared dates only.') print('\twith following dates are ignored for differencing:\n{}'.format(dateExcluded)) dateShared[np.array([obj1.dateList.index(i) for i in dateExcluded])] = 0 else: raise Exception('To enforce the differencing anyway, use --force option.') # consider different reference_date/pixel data2 = readfile.read(file2[0], datasetName=dateListShared)[0] * unit_fac if ref_date: data2 -= np.tile(data2[obj2.dateList.index(ref_date), :, :], (data2.shape[0], 1, 1)) if ref_y and ref_x: data2 -= np.tile(data2[:, ref_y, ref_x].reshape(-1, 1, 1), (1, data2.shape[1], data2.shape[2])) data = obj1.read() mask = data == 0. data[dateShared] -= data2 data[mask] = 0. # Do not change zero phase value del data2 writefile.write(data, out_file=outFile, ref_file=file1) elif all(i == 'ifgramStack' for i in [k1, k2]): obj1 = ifgramStack(file1) obj1.open() obj2 = ifgramStack(file2[0]) obj2.open() dsNames = list(set(obj1.datasetNames) & set(obj2.datasetNames)) if len(dsNames) == 0: raise ValueError('no common dataset between two files!') dsName = [i for i in ifgramDatasetNames if i in dsNames][0] # read data print('reading {} from file {} ...'.format(dsName, file1)) data1 = readfile.read(file1, datasetName=dsName)[0] print('reading {} from file {} ...'.format(dsName, file2[0])) data2 = readfile.read(file2[0], datasetName=dsName)[0] # consider reference pixel if 'unwrapphase' in dsName.lower(): print('referencing to pixel ({},{}) ...'.format(obj1.refY, obj1.refX)) ref1 = data1[:, obj1.refY, obj1.refX] ref2 = data2[:, obj2.refY, obj2.refX] for i in range(data1.shape[0]): data1[i,:][data1[i, :] != 0.] -= ref1[i] data2[i,:][data2[i, :] != 0.] -= ref2[i] # operation and ignore zero values data1[data1 == 0] = np.nan data2[data2 == 0] = np.nan data = data1 - data2 del data1, data2 data[np.isnan(data)] = 0. # write to file dsDict = {} dsDict[dsName] = data writefile.write(dsDict, out_file=outFile, ref_file=file1) # Sing dataset file else: data1 = readfile.read(file1)[0] data = np.array(data1, data1.dtype) for fname in file2: data2 = readfile.read(fname)[0] data = np.array(data, dtype=np.float32) - np.array(data2, dtype=np.float32) data = np.array(data, data1.dtype) print('writing >>> '+outFile) writefile.write(data, out_file=outFile, metadata=atr1) return outFile
def diff_file(file1, file2, outFile=None, force=False): """Subtraction/difference of two input files""" if not outFile: fbase, fext = os.path.splitext(file1) if len(file2) > 1: raise ValueError( 'Output file name is needed for more than 2 files input.') outFile = '{}_diff_{}{}'.format( fbase, os.path.splitext(os.path.basename(file2[0]))[0], fext) print('{} - {} --> {}'.format(file1, file2, outFile)) # Read basic info atr1 = readfile.read_attribute(file1) k1 = atr1['FILE_TYPE'] atr2 = readfile.read_attribute(file2[0]) k2 = atr2['FILE_TYPE'] print('input files are: {} and {}'.format(k1, k2)) if k1 == 'timeseries': if k2 not in ['timeseries', 'giantTimeseries']: raise Exception( 'Input multiple dataset files are not the same file type!') if len(file2) > 1: raise Exception( ('Only 2 files substraction is supported for time series file,' ' {} input.'.format(len(file2) + 1))) obj1 = timeseries(file1) obj1.open() if k2 == 'timeseries': obj2 = timeseries(file2[0]) unit_fac = 1. elif k2 == 'giantTimeseries': obj2 = giantTimeseries(file2[0]) unit_fac = 0.001 obj2.open() ref_date, ref_y, ref_x = _check_reference(obj1.metadata, obj2.metadata) # check dates shared by two timeseries files dateListShared = [i for i in obj1.dateList if i in obj2.dateList] dateShared = np.ones((obj1.numDate), dtype=np.bool_) if dateListShared != obj1.dateList: print('WARNING: {} does not contain all dates in {}'.format( file2, file1)) if force: dateExcluded = list(set(obj1.dateList) - set(dateListShared)) print( 'Continue and enforce the differencing for their shared dates only.' ) print( '\twith following dates are ignored for differencing:\n{}'. format(dateExcluded)) dateShared[np.array( [obj1.dateList.index(i) for i in dateExcluded])] = 0 else: raise Exception( 'To enforce the differencing anyway, use --force option.') # consider different reference_date/pixel data2 = readfile.read(file2[0], datasetName=dateListShared)[0] * unit_fac if ref_date: data2 -= np.tile(data2[obj2.dateList.index(ref_date), :, :], (data2.shape[0], 1, 1)) if ref_y and ref_x: data2 -= np.tile(data2[:, ref_y, ref_x].reshape(-1, 1, 1), (1, data2.shape[1], data2.shape[2])) data = obj1.read() mask = data == 0. data[dateShared] -= data2 data[mask] = 0. # Do not change zero phase value writefile.write(data, out_file=outFile, ref_file=file1) # Sing dataset file else: data1 = readfile.read(file1)[0] data = np.array(data1, data1.dtype) for fname in file2: data2 = readfile.read(fname)[0] data = np.array(data, dtype=np.float32) - np.array( data2, dtype=np.float32) data = np.array(data, data1.dtype) print('writing >>> ' + outFile) writefile.write(data, out_file=outFile, metadata=atr1) return outFile