def update_object(outFile, inObj, box, updateMode=True): """Do not write h5 file if: 1) h5 exists and readable, 2) it contains all date12 from ifgramStackDict, or all datasets from geometryDict""" write_flag = True if updateMode and ut.run_or_skip(outFile, check_readable=True) == 'skip': if inObj.name == 'ifgramStack': in_size = inObj.get_size(box=box)[1:] in_date12_list = inObj.get_date12_list() outObj = ifgramStack(outFile) out_size = outObj.get_size()[1:] out_date12_list = outObj.get_date12_list(dropIfgram=False) if out_size == in_size and set(in_date12_list).issubset( set(out_date12_list)): print(( 'All date12 exists in file {} with same size as required,' ' no need to re-load.'.format(outFile))) write_flag = False elif inObj.name == 'geometry': outObj = geometry(outFile) outObj.open(print_msg=False) if (outObj.get_size() == inObj.get_size(box=box) and all(i in outObj.datasetNames for i in inObj.get_dataset_list())): print(( 'All datasets exists in file {} with same size as required,' ' no need to re-load.'.format(outFile))) write_flag = False return write_flag
def update_object(outFile, inObj, box, updateMode=True): """Do not write h5 file if: 1) h5 exists and readable, 2) it contains all date12 from ifgramStackDict, or all datasets from geometryDict""" updateFile = True if updateMode and not ut.update_file(outFile, check_readable=True): if inObj.name == 'ifgramStack': outObj = ifgramStack(outFile) if (outObj.get_size() == inObj.get_size(box=box) and sorted(outObj.get_date12_list(dropIfgram=False)) == sorted(inObj.get_date12_list())): print(( 'All date12 exists in file {} with same size as required,' ' no need to re-load.'.format(outFile))) updateFile = False elif inObj.name == 'geometry': outObj = geometry(outFile) outObj.open(print_msg=False) if (outObj.get_size() == inObj.get_size(box=box) and all(i in outObj.datasetNames for i in inObj.get_dataset_list())): print(( 'All datasets exists in file{} with same size as required,' ' no need to re-load.'.format(outFile))) updateFile = False return updateFile
def read_geometry(inps): ts_obj = timeseries(inps.timeseries_file) ts_obj.open(print_msg=False) # 2D / 3D geometry if inps.geom_file: geom_obj = geometry(inps.geom_file) geom_obj.open() print(('read 2D incidenceAngle,slantRangeDistance from {} file:' ' {}').format(geom_obj.name, os.path.basename(geom_obj.file))) inps.incAngle = geom_obj.read(datasetName='incidenceAngle', print_msg=False).flatten() inps.rangeDist = geom_obj.read(datasetName='slantRangeDistance', print_msg=False).flatten() if 'bperp' in geom_obj.datasetNames: print('read 3D bperp from {} file: {} ...'.format(geom_obj.name, os.path.basename(geom_obj.file))) inps.pbase = geom_obj.read(datasetName='bperp', print_msg=False).reshape((geom_obj.numDate, -1)) inps.pbase -= inps.pbase[ts_obj.refIndex] else: print('read mean bperp from {} file'.format(ts_obj.name)) inps.pbase = ts_obj.pbase.reshape((-1, 1)) # 0D geometry else: print('read mean incidenceAngle,slantRangeDistance,bperp value from {} file'.format(ts_obj.name)) inps.incAngle = ut.incidence_angle(ts_obj.metadata, dimension=0) inps.rangeDist = ut.range_distance(ts_obj.metadata, dimension=0) inps.pbase = ts_obj.pbase.reshape((-1, 1)) inps.sinIncAngle = np.sin(inps.incAngle * np.pi / 180.) return inps
def check_loaded_dataset(work_dir='./', print_msg=True): """Check the result of loading data for the following two rules: 1. file existance 2. file attribute readability Parameters: work_dir : string, PySAR working directory print_msg : bool, print out message Returns: True, if all required files and dataset exist; otherwise, ERROR If True, PROCESS, SLC folder could be removed. stack_file : geom_file : lookup_file : Example: work_dir = os.path.expandvars('$SCRATCHDIR/SinabungT495F50AlosA/PYSAR') ut.check_loaded_dataset(work_dir) """ load_complete = True if not work_dir: work_dir = os.getcwd() work_dir = os.path.abspath(work_dir) # 1. interferograms stack file: unwrapPhase, coherence flist = [os.path.join(work_dir, 'INPUTS/ifgramStack.h5')] stack_file = is_file_exist(flist, abspath=True) if stack_file is not None: obj = ifgramStack(stack_file) obj.open(print_msg=False) for dname in ['unwrapPhase', 'coherence']: if dname not in obj.datasetNames: raise ValueError( 'required dataset "{}" is missing in file {}'.format( dname, stack_file)) else: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), './INPUTS/ifgramStack.h5') atr = readfile.read_attribute(stack_file) # 2. geom_file: height if 'X_FIRST' in atr.keys(): flist = [os.path.join(work_dir, 'INPUTS/geometryGeo.h5')] else: flist = [os.path.join(work_dir, 'INPUTS/geometryRadar.h5')] geom_file = is_file_exist(flist, abspath=True) if geom_file is not None: obj = geometry(geom_file) obj.open(print_msg=False) dname = geometryDatasetNames[0] if dname not in obj.datasetNames: raise ValueError( 'required dataset "{}" is missing in file {}'.format( dname, geom_file)) else: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), './INPUTS/geometry*.h5') # 3. lookup_file: latitude,longitude or rangeCoord,azimuthCoord # could be different than geometry file in case of roipac and gamma flist = [os.path.join(work_dir, 'INPUTS/geometry*.h5')] lookup_file = get_lookup_file(flist, abspath=True, print_msg=print_msg) if lookup_file is not None: obj = geometry(lookup_file) obj.open(print_msg=False) if atr['PROCESSOR'] in ['isce', 'doris']: dnames = [geometryDatasetNames[1], geometryDatasetNames[2]] elif atr['PROCESSOR'] in ['gamma', 'roipac']: dnames = [geometryDatasetNames[3], geometryDatasetNames[4]] else: raise AttributeError('InSAR processor: {}'.format( atr['PROCESSOR'])) for dname in dnames: if dname not in obj.datasetNames: load_complete = False raise Exception( 'required dataset "{}" is missing in file {}'.format( dname, lookup_file)) else: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), './INPUTS/geometry*.h5') # print message if print_msg: print(('Loaded dataset are processed by ' 'InSAR software: {}'.format(atr['PROCESSOR']))) if 'X_FIRST' in atr.keys(): print('Loaded dataset is in GEO coordinates') else: print('Loaded dataset is in RADAR coordinates') print('Interferograms Stack: {}'.format(stack_file)) print('Geometry File : {}'.format(geom_file)) print('Lookup Table File : {}'.format(lookup_file)) if load_complete: print('-' * 50) print( 'All data needed found/loaded/copied. Processed 2-pass InSAR data can be removed.' ) print('-' * 50) return load_complete, stack_file, geom_file, lookup_file
def get_slice_list(fname): """Get list of 2D slice existed in file (for display)""" fbase, fext = os.path.splitext(os.path.basename(fname)) fext = fext.lower() atr = read_attribute(fname) k = atr['FILE_TYPE'] global slice_list # HDF5 Files if fext in ['.h5', '.he5']: with h5py.File(fname, 'r') as f: d1_list = [i for i in f.keys() if isinstance(f[i], h5py.Dataset)] if k == 'timeseries' and k in d1_list: obj = timeseries(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['geometry'] and k not in d1_list: obj = geometry(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['ifgramStack']: obj = ifgramStack(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['HDFEOS']: obj = HDFEOS(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['giantTimeseries']: obj = giantTimeseries(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['giantIfgramStack']: obj = giantIfgramStack(fname) obj.open(print_msg=False) slice_list = obj.sliceList else: ## Find slice by walking through the file structure length, width = int(atr['LENGTH']), int(atr['WIDTH']) def get_hdf5_2d_dataset(name, obj): global slice_list if isinstance(obj, h5py.Dataset) and obj.shape[-2:] == (length, width): if obj.ndim == 2: slice_list.append(name) else: warnings.warn( 'file has un-defined {}D dataset: {}'.format( obj.ndim, name)) slice_list = [] with h5py.File(fname, 'r') as f: f.visititems(get_hdf5_2d_dataset) # Binary Files else: if fext.lower() in ['.trans', '.utm_to_rdc']: slice_list = ['rangeCoord', 'azimuthCoord'] elif fbase.startswith('los'): slice_list = ['incidenceAngle', 'azimuthAngle'] elif atr.get('number_bands', '1') == '2' and 'unw' not in k: slice_list = ['band1', 'band2'] else: slice_list = [''] return slice_list
def get_delay_timeseries(inps, atr): """Calculate delay time-series and write it to HDF5 file. Parameters: inps : namespace, all input parameters atr : dict, metadata to be saved in trop_file Returns: trop_file : str, file name of ECMWF.h5 """ def get_dataset_size(fname): atr = readfile.read_attribute(fname) return (atr['LENGTH'], atr['WIDTH']) # check 1 - existing tropo delay file if (ut.run_or_skip(out_file=inps.trop_file, in_file=inps.grib_file_list, print_msg=False) == 'skip' and get_dataset_size(inps.trop_file) == get_dataset_size( inps.geom_file)): print( '{} file exists and is newer than all GRIB files, skip updating.'. format(inps.trop_file)) return # check 2 - geometry file if any(i is None for i in [inps.geom_file, inps.ref_yx]): print( 'No DEM / incidenceAngle / ref_yx found, skip calculating tropospheric delays.' ) if not os.path.isfile(inps.trop_file): inps.trop_file = None return # prepare geometry data geom_obj = geometry(inps.geom_file) geom_obj.open() inps.dem = geom_obj.read(datasetName='height') inps.inc = geom_obj.read(datasetName='incidenceAngle') if 'latitude' in geom_obj.datasetNames: inps.lat = geom_obj.read(datasetName='latitude') inps.lon = geom_obj.read(datasetName='longitude') else: inps.lat, inps.lon = get_lat_lon(geom_obj.metadata) # calculate phase delay length, width = int(atr['LENGTH']), int(atr['WIDTH']) num_date = len(inps.grib_file_list) date_list = [str(re.findall('\d{8}', i)[0]) for i in inps.grib_file_list] trop_data = np.zeros((num_date, length, width), np.float32) print( 'calcualting delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...' ) print('number of grib files used: {}'.format(num_date)) if verbose: prog_bar = ptime.progressBar(maxValue=num_date) for i in range(num_date): grib_file = inps.grib_file_list[i] trop_data[i] = get_delay(grib_file, inps) if verbose: prog_bar.update(i + 1, suffix=os.path.basename(grib_file)) if verbose: prog_bar.close() # Convert relative phase delay on reference date inps.ref_date = atr.get('REF_DATE', date_list[0]) print('convert to relative phase delay with reference date: ' + inps.ref_date) inps.ref_idx = date_list.index(inps.ref_date) trop_data -= np.tile(trop_data[inps.ref_idx, :, :], (num_date, 1, 1)) # Write tropospheric delay to HDF5 atr['REF_Y'] = inps.ref_yx[0] atr['REF_X'] = inps.ref_yx[1] ts_obj = timeseries(inps.trop_file) ts_obj.write2hdf5(data=trop_data, dates=date_list, metadata=atr, refFile=inps.timeseries_file) return
def write2hdf5(out_file, ts_file, coh_file, mask_file, geom_file, metadata): """Write HDF5 file in HDF-EOS5 format""" ts_obj = timeseries(ts_file) ts_obj.open(print_msg=False) dateList = ts_obj.dateList # Open HDF5 File f = h5py.File(out_file, 'w') print('create HDF5 file: {} with w mode'.format(out_file)) maxDigit = 20 # Write Observation - Displacement gName = 'HDFEOS/GRIDS/timeseries/observation' print('create group /{}'.format(gName)) group = f.create_group(gName) dsName = 'displacement' data = ts_obj.read(print_msg=False) print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}' ' with compression={c}').format(g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape, c=compression)) dset = group.create_dataset(dsName, data=data, dtype=np.float32, chunks=True, compression=compression) dset.attrs['Title'] = dsName dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = 'meters' dsName = 'date' data = np.array(dateList, dtype=np.string_) group.create_dataset(dsName, data=data) print('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'.format( g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape)) dsName = 'bperp' data = np.array(ts_obj.pbase, dtype=np.float32) group.create_dataset(dsName, data=data) print('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'.format( g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape)) # Write Quality gName = 'HDFEOS/GRIDS/timeseries/quality' print('create group /{}'.format(gName)) group = f.create_group(gName) ## 1 - temporalCoherence dsName = 'temporalCoherence' data = readfile.read(coh_file)[0] print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}' ' with compression={c}').format(g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape, c=compression)) dset = group.create_dataset(dsName, data=data, chunks=True, compression=compression) dset.attrs['Title'] = dsName dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = '1' ## 2 - mask dsName = 'mask' data = readfile.read(mask_file, datasetName='mask')[0] print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}' ' with compression={c}').format(g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape, c=compression)) dset = group.create_dataset(dsName, data=data, chunks=True, compression=compression) dset.attrs['Title'] = dsName dset.attrs['MissingValue'] = BOOL_ZERO dset.attrs['_FillValue'] = BOOL_ZERO dset.attrs['Units'] = '1' # Write Geometry # Required: height, incidenceAngle # Optional: rangeCoord, azimuthCoord, azimuthAngle, slantRangeDistance, waterMask, shadowMask gName = 'HDFEOS/GRIDS/timeseries/geometry' print('create group /{}'.format(gName)) group = f.create_group(gName) geom_obj = geometry(geom_file) geom_obj.open(print_msg=False) for dsName in geom_obj.datasetNames: data = geom_obj.read(datasetName=dsName, print_msg=False) print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}' ' with compression={c}').format(g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape, c=compression)) dset = group.create_dataset(dsName, data=data, chunks=True, compression=compression) dset.attrs['Title'] = dsName if dsName in ['height', 'slantRangeDistance', 'bperp']: dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = 'meters' elif dsName in [ 'incidenceAngle', 'azimuthAngle', 'latitude', 'longitude' ]: dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = 'degrees' elif dsName in ['rangeCoord', 'azimuthCoord']: dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = '1' elif dsName in ['waterMask', 'shadowMask']: dset.attrs['MissingValue'] = BOOL_ZERO dset.attrs['_FillValue'] = BOOL_ZERO dset.attrs['Units'] = '1' # Write Attributes to the HDF File print('write metadata to root level') for key, value in iter(metadata.items()): f.attrs[key] = value f.close() print('finished writing to {}'.format(out_file)) return out_file