def update_object(outFile, inObj, box, updateMode=True): """Do not write h5 file if: 1) h5 exists and readable, 2) it contains all date12 from ifgramStackDict, or all datasets from geometryDict""" write_flag = True if updateMode and ut.run_or_skip(outFile, check_readable=True) == 'skip': if inObj.name == 'ifgramStack': in_size = inObj.get_size(box=box)[1:] in_date12_list = inObj.get_date12_list() outObj = ifgramStack(outFile) out_size = outObj.get_size()[1:] out_date12_list = outObj.get_date12_list(dropIfgram=False) if out_size == in_size and set(in_date12_list).issubset(set(out_date12_list)): print(('All date12 exists in file {} with same size as required,' ' no need to re-load.'.format(os.path.basename(outFile)))) write_flag = False elif inObj.name == 'geometry': outObj = geometry(outFile) outObj.open(print_msg=False) if (outObj.get_size() == inObj.get_size(box=box) and all(i in outObj.datasetNames for i in inObj.get_dataset_list())): print(('All datasets exists in file {} with same size as required,' ' no need to re-load.'.format(os.path.basename(outFile)))) write_flag = False return write_flag
def read_geometry(inps): ts_obj = timeseries(inps.timeseries_file) ts_obj.open(print_msg=False) # 2D / 3D geometry if inps.geom_file: geom_obj = geometry(inps.geom_file) geom_obj.open() if 'incidenceAngle' not in geom_obj.datasetNames: inps.incAngle = ut.incidence_angle(ts_obj.metadata, dimension=0) inps.rangeDist = ut.range_distance(ts_obj.metadata, dimension=0) else: print(('read 2D incidenceAngle,slantRangeDistance from {} file:' ' {}').format(geom_obj.name, os.path.basename(geom_obj.file))) inps.incAngle = geom_obj.read(datasetName='incidenceAngle', print_msg=False).flatten() inps.rangeDist = geom_obj.read(datasetName='slantRangeDistance', print_msg=False).flatten() if 'bperp' in geom_obj.datasetNames: print('read 3D bperp from {} file: {} ...'.format(geom_obj.name, os.path.basename(geom_obj.file))) dset_list = ['bperp-{}'.format(d) for d in ts_obj.dateList] inps.pbase = geom_obj.read(datasetName=dset_list, print_msg=False).reshape((ts_obj.numDate, -1)) inps.pbase -= np.tile(inps.pbase[ts_obj.refIndex, :].reshape(1, -1), (ts_obj.numDate, 1)) else: print('read mean bperp from {} file'.format(ts_obj.name)) inps.pbase = ts_obj.pbase.reshape((-1, 1)) # 0D geometry else: print('read mean incidenceAngle,slantRangeDistance,bperp value from {} file'.format(ts_obj.name)) inps.incAngle = ut.incidence_angle(ts_obj.metadata, dimension=0) inps.rangeDist = ut.range_distance(ts_obj.metadata, dimension=0) inps.pbase = ts_obj.pbase.reshape((-1, 1)) inps.sinIncAngle = np.sin(inps.incAngle * np.pi / 180.) return inps
def run_or_skip(inps, dsNameDict, out_file): flag = 'run' # check 1 - update mode status if not inps.updateMode: return flag # check 2 - output file existance if ut.run_or_skip(out_file, check_readable=True) == 'run': return flag # check 3 - output dataset info key = [i for i in ['unwrapPhase', 'height'] if i in dsNameDict.keys()][0] ds_shape = dsNameDict[key][1] in_shape = ds_shape[-2:] if 'unwrapPhase' in dsNameDict.keys(): # compare date12 and size ds = gdal.Open(inps.unwFile, gdal.GA_ReadOnly) in_date12_list = [ ds.GetRasterBand(i + 1).GetMetadata("unwrappedPhase")['Dates'] for i in range(ds_shape[0]) ] in_date12_list = ['_'.join(d.split('_')[::-1]) for d in in_date12_list] try: out_obj = ifgramStack(out_file) out_obj.open(print_msg=False) out_shape = (out_obj.length, out_obj.width) out_date12_list = out_obj.get_date12_list(dropIfgram=False) if out_shape == in_shape and set(in_date12_list).issubset( set(out_date12_list)): print(( 'All date12 exists in file {} with same size as required,' ' no need to re-load.'.format(os.path.basename(out_file)))) flag = 'skip' except: pass elif 'height' in dsNameDict.keys(): # compare dataset names and size in_dsNames = list(dsNameDict.keys()) in_size = in_shape[0] * in_shape[1] * 4 * len(in_dsNames) out_obj = geometry(out_file) out_obj.open(print_msg=False) out_dsNames = out_obj.datasetNames out_shape = (out_obj.length, out_obj.width) out_size = os.path.getsize(out_file) if (set(in_dsNames).issubset(set(out_dsNames)) and out_shape == in_shape and out_size > in_size * 0.3): print(('All datasets exists in file {} with same size as required,' ' no need to re-load.'.format(os.path.basename(out_file)))) flag = 'skip' return flag
def read_geometry(ts_file, geom_file=None, box=None): """Read the following geometry info in 0/2/3D Parameters: ts_file - str, path of time-series file geom_file - str, path of geometry file box - tuple of 4 int for (x0, y0, x1, y1) of the area of interest Returns: sin_inc_angle - 0/2D array, sin(inc_angle) range_dist - 0/2D array, slant range distance in meter pbase - 0/3D array, perp baseline in meter """ ts_obj = timeseries(ts_file) ts_obj.open(print_msg=False) # 0/2/3D geometry if geom_file: geom_obj = geometry(geom_file) geom_obj.open() # 0/2D incidence angle / slant range distance if 'incidenceAngle' not in geom_obj.datasetNames: inc_angle = ut.incidence_angle(ts_obj.metadata, dimension=0) range_dist = ut.range_distance(ts_obj.metadata, dimension=0) else: print( 'read 2D incidenceAngle, slantRangeDistance from {} file: {}'. format(geom_obj.name, os.path.basename(geom_obj.file))) inc_angle = geom_obj.read(datasetName='incidenceAngle', box=box, print_msg=False).flatten() range_dist = geom_obj.read(datasetName='slantRangeDistance', box=box, print_msg=False).flatten() # 0/3D perp baseline if 'bperp' in geom_obj.datasetNames: print('read 3D bperp from {} file: {} ...'.format( geom_obj.name, os.path.basename(geom_obj.file))) dset_list = ['bperp-{}'.format(d) for d in ts_obj.dateList] pbase = geom_obj.read(datasetName=dset_list, box=box, print_msg=False).reshape( (ts_obj.numDate, -1)) pbase -= np.tile(pbase[ts_obj.refIndex, :].reshape(1, -1), (ts_obj.numDate, 1)) else: print('read mean bperp from {} file'.format(ts_obj.name)) pbase = ts_obj.pbase.reshape((-1, 1)) # 0D geometry else: print( 'read mean incidenceAngle, slantRangeDistance, bperp value from {} file' .format(ts_obj.name)) inc_angle = ut.incidence_angle(ts_obj.metadata, dimension=0) range_dist = ut.range_distance(ts_obj.metadata, dimension=0) pbase = ts_obj.pbase.reshape((-1, 1)) sin_inc_angle = np.sin(inc_angle * np.pi / 180.) return sin_inc_angle, range_dist, pbase
def split2boxes(ts_file, geom_file=None, memory_size=4, num_step=0, print_msg=True): """Split into chunks in rows to reduce memory usage Parameters: ts_file - str, path of time-series h5 file memory_size - float, max memory to use in GB print_msg - bool Returns: box_list - list of tuple of 4 int num_box - int, number of boxes """ ts_obj = timeseries(ts_file) ts_obj.open(print_msg=False) length, width = ts_obj.length, ts_obj.width # 1st dimension size: ts (obs / cor / res / step) + dem_err/inc_angle/rg_dist (+pbase) num_epoch = ts_obj.numDate * 3 + num_step + 3 if geom_file: geom_obj = geometry(geom_file) geom_obj.open(print_msg=False) if 'bperp' in geom_obj.datasetNames: num_epoch += ts_obj.numDate # split in lines based on the input memory limit y_step = (memory_size * (1e3**3)) / (num_epoch * width * 4) # calibrate based on experience y_step = int(ut.round_to_1(y_step * 0.7)) num_box = int((length - 1) / y_step) + 1 if print_msg and num_box > 1: print('maximum memory size: %.1E GB' % memory_size) print('split %d lines into %d patches for processing' % (length, num_box)) print(' with each patch up to %d lines' % y_step) # y_step / num_box --> box_list box_list = [] for i in range(num_box): y0 = i * y_step y1 = min([length, y0 + y_step]) box = (0, y0, width, y1) box_list.append(box) return box_list, num_box
def get_delay_timeseries(inps, atr): """Calculate delay time-series and write it to HDF5 file. Parameters: inps : namespace, all input parameters atr : dict, metadata to be saved in trop_file Returns: trop_file : str, file name of ECMWF.h5 """ def get_dataset_size(fname): atr = readfile.read_attribute(fname) return (atr['LENGTH'], atr['WIDTH']) # check 1 - existing tropo delay file if (ut.run_or_skip(out_file=inps.trop_file, in_file=inps.grib_file_list, print_msg=False) == 'skip' and get_dataset_size(inps.trop_file) == get_dataset_size(inps.geom_file)): print('{} file exists and is newer than all GRIB files, skip updating.'.format(inps.trop_file)) return # check 2 - geometry file if any(i is None for i in [inps.geom_file, inps.ref_yx]): print('No DEM / incidenceAngle / ref_yx found, skip calculating tropospheric delays.') if not os.path.isfile(inps.trop_file): inps.trop_file = None return # prepare geometry data geom_obj = geometry(inps.geom_file) geom_obj.open() inps.dem = geom_obj.read(datasetName='height') inps.inc = geom_obj.read(datasetName='incidenceAngle') if 'latitude' in geom_obj.datasetNames: inps.lat = geom_obj.read(datasetName='latitude') inps.lon = geom_obj.read(datasetName='longitude') else: inps.lat, inps.lon = get_lat_lon(geom_obj.metadata) # calculate phase delay length, width = int(atr['LENGTH']), int(atr['WIDTH']) num_date = len(inps.grib_file_list) date_list = [str(re.findall('\d{8}', i)[0]) for i in inps.grib_file_list] trop_data = np.zeros((num_date, length, width), np.float32) #trop_data_abs = np.zeros((num_date, length, width), np.float32) print('calcualting delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...') print('number of grib files used: {}'.format(num_date)) if verbose: prog_bar = ptime.progressBar(maxValue=num_date) for i in range(num_date): grib_file = inps.grib_file_list[i] trop_data[i] = get_delay(grib_file, inps) #trop_data_abs[i] = get_delay_abs(grib_file, inps) if verbose: prog_bar.update(i+1, suffix=os.path.basename(grib_file)) if verbose: prog_bar.close() # Convert relative phase delay on reference date inps.ref_date = atr.get('REF_DATE', date_list[0]) print('convert to relative phase delay with reference date: '+inps.ref_date) inps.ref_idx = date_list.index(inps.ref_date) trop_data -= np.tile(trop_data[inps.ref_idx, :, :], (num_date, 1, 1)) # Write tropospheric delay to HDF5 atr['REF_Y'] = inps.ref_yx[0] atr['REF_X'] = inps.ref_yx[1] ts_obj = timeseries(inps.trop_file) ts_obj.write2hdf5(data=trop_data, dates=date_list, metadata=atr, refFile=inps.timeseries_file) #ts_obj.write2hdf5(data=trop_data, # outFile = 'ERA5_abs.h5', # dates=date_list, # metadata=atr, # refFile=inps.timeseries_file) return
def get_delay_timeseries(inps, atr): """Calculate delay time-series and write it to HDF5 file. Parameters: inps : namespace, all input parameters atr : dict, metadata to be saved in trop_file Returns: trop_file : str, file name of ECMWF.h5 """ def get_dataset_size(fname): atr = readfile.read_attribute(fname) return (atr['LENGTH'], atr['WIDTH']) # check 1 - existing tropo delay file if (ut.run_or_skip(out_file=inps.trop_file, in_file=inps.grib_file_list, print_msg=False) == 'skip' and get_dataset_size(inps.trop_file) == get_dataset_size(inps.geom_file)): print('{} file exists and is newer than all GRIB files, skip updating.'.format(inps.trop_file)) return # check 2 - geometry file if any(i is None for i in [inps.geom_file, inps.ref_yx]): print('No DEM / incidenceAngle / ref_yx found, skip calculating tropospheric delays.') if not os.path.isfile(inps.trop_file): inps.trop_file = None return # prepare geometry data geom_obj = geometry(inps.geom_file) geom_obj.open() inps.dem = geom_obj.read(datasetName='height') inps.inc = geom_obj.read(datasetName='incidenceAngle') if 'latitude' in geom_obj.datasetNames: inps.lat = geom_obj.read(datasetName='latitude') inps.lon = geom_obj.read(datasetName='longitude') else: inps.lat, inps.lon = get_lat_lon(geom_obj.metadata) # calculate phase delay length, width = int(atr['LENGTH']), int(atr['WIDTH']) num_date = len(inps.grib_file_list) date_list = [str(re.findall('\d{8}', i)[0]) for i in inps.grib_file_list] trop_data = np.zeros((num_date, length, width), np.float32) print('calcualting delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...') print('number of grib files used: {}'.format(num_date)) prog_bar = ptime.progressBar(maxValue=num_date) for i in range(num_date): grib_file = inps.grib_file_list[i] trop_data[i] = get_delay(grib_file, inps) prog_bar.update(i+1, suffix=os.path.basename(grib_file)) prog_bar.close() # Convert relative phase delay on reference date inps.ref_date = atr.get('REF_DATE', date_list[0]) print('convert to relative phase delay with reference date: '+inps.ref_date) inps.ref_idx = date_list.index(inps.ref_date) trop_data -= np.tile(trop_data[inps.ref_idx, :, :], (num_date, 1, 1)) # Write tropospheric delay to HDF5 atr['REF_Y'] = inps.ref_yx[0] atr['REF_X'] = inps.ref_yx[1] ts_obj = timeseries(inps.trop_file) ts_obj.write2hdf5(data=trop_data, dates=date_list, metadata=atr, refFile=inps.timeseries_file) return
def get_slice_list(fname): """Get list of 2D slice existed in file (for display)""" fbase, fext = os.path.splitext(os.path.basename(fname)) fext = fext.lower() atr = read_attribute(fname) k = atr['FILE_TYPE'] global slice_list # HDF5 Files if fext in ['.h5', '.he5']: with h5py.File(fname, 'r') as f: d1_list = [i for i in f.keys() if isinstance(f[i], h5py.Dataset)] if k == 'timeseries' and k in d1_list: obj = timeseries(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['geometry'] and k not in d1_list: obj = geometry(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['ifgramStack']: obj = ifgramStack(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['HDFEOS']: obj = HDFEOS(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['giantTimeseries']: obj = giantTimeseries(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['giantIfgramStack']: obj = giantIfgramStack(fname) obj.open(print_msg=False) slice_list = obj.sliceList else: ## Find slice by walking through the file structure length, width = int(atr['LENGTH']), int(atr['WIDTH']) def get_hdf5_2d_dataset(name, obj): global slice_list if isinstance(obj, h5py.Dataset) and obj.shape[-2:] == (length, width): if obj.ndim == 2: slice_list.append(name) else: warnings.warn( 'file has un-defined {}D dataset: {}'.format( obj.ndim, name)) slice_list = [] with h5py.File(fname, 'r') as f: f.visititems(get_hdf5_2d_dataset) # Binary Files else: if fext.lower() in ['.trans', '.utm_to_rdc']: slice_list = ['rangeCoord', 'azimuthCoord'] elif fbase.startswith('los'): slice_list = ['incidenceAngle', 'azimuthAngle'] elif atr.get('number_bands', '1') == '2' and 'unw' not in k: slice_list = ['band1', 'band2'] else: slice_list = [''] return slice_list
def get_slice_list(fname): """Get list of 2D slice existed in file (for display)""" fbase, fext = os.path.splitext(os.path.basename(fname)) fext = fext.lower() atr = read_attribute(fname) k = atr['FILE_TYPE'] global slice_list # HDF5 Files if fext in ['.h5', '.he5']: with h5py.File(fname, 'r') as f: d1_list = [i for i in f.keys() if isinstance(f[i], h5py.Dataset)] if k == 'timeseries' and k in d1_list: obj = timeseries(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['geometry'] and k not in d1_list: obj = geometry(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['ifgramStack']: obj = ifgramStack(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['HDFEOS']: obj = HDFEOS(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['giantTimeseries']: obj = giantTimeseries(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['giantIfgramStack']: obj = giantIfgramStack(fname) obj.open(print_msg=False) slice_list = obj.sliceList else: ## Find slice by walking through the file structure length, width = int(atr['LENGTH']), int(atr['WIDTH']) def get_hdf5_2d_dataset(name, obj): global slice_list if isinstance(obj, h5py.Dataset) and obj.shape[-2:] == (length, width): if obj.ndim == 2: slice_list.append(name) else: warnings.warn('file has un-defined {}D dataset: {}'.format(obj.ndim, name)) slice_list = [] with h5py.File(fname, 'r') as f: f.visititems(get_hdf5_2d_dataset) # Binary Files else: if fext.lower() in ['.trans', '.utm_to_rdc']: slice_list = ['rangeCoord', 'azimuthCoord'] elif fbase.startswith('los'): slice_list = ['incidenceAngle', 'azimuthAngle'] elif atr.get('number_bands', '1') == '2' and 'unw' not in k: slice_list = ['band1', 'band2'] else: slice_list = [''] return slice_list
def check_inputs(inps): parser = create_parser() # output directories/files atr = dict() mintpy_dir = None if inps.timeseries_file: atr = readfile.read_attribute(inps.timeseries_file) mintpy_dir = os.path.dirname(inps.timeseries_file) if not inps.outfile: fbase = os.path.splitext(inps.timeseries_file)[0] inps.outfile = '{}_{}.h5'.format(fbase, inps.trop_model) elif inps.geom_file: atr = readfile.read_attribute(inps.geom_file) mintpy_dir = os.path.join(os.path.dirname(inps.geom_file), '..') else: mintpy_dir = os.path.abspath(os.getcwd()) # trop_file inps.trop_file = os.path.join(mintpy_dir, 'inputs/{}.h5'.format(inps.trop_model)) print('output tropospheric delay file: {}'.format(inps.trop_file)) # hour if not inps.hour: if 'CENTER_LINE_UTC' in atr.keys(): inps.hour = ptime.closest_weather_product_time(atr['CENTER_LINE_UTC'], inps.trop_model) else: parser.print_usage() raise Exception('no input for hour') print('time of cloest available product: {}:00 UTC'.format(inps.hour)) # date list if inps.timeseries_file: print('read date list from timeseries file: {}'.format(inps.timeseries_file)) ts_obj = timeseries(inps.timeseries_file) ts_obj.open(print_msg=False) inps.date_list = ts_obj.dateList elif len(inps.date_list) == 1: if os.path.isfile(inps.date_list[0]): print('read date list from text file: {}'.format(inps.date_list[0])) inps.date_list = ptime.yyyymmdd(np.loadtxt(inps.date_list[0], dtype=bytes, usecols=(0,)).astype(str).tolist()) else: parser.print_usage() raise Exception('ERROR: input date list < 2') # Grib data directory inps.grib_dir = os.path.join(inps.weather_dir, inps.trop_model) if not os.path.isdir(inps.grib_dir): os.makedirs(inps.grib_dir) print('making directory: '+inps.grib_dir) # Date list to grib file list inps.grib_file_list = date_list2grib_file(inps.date_list, inps.hour, inps.trop_model, inps.grib_dir) if 'REF_Y' in atr.keys(): inps.ref_yx = [int(atr['REF_Y']), int(atr['REF_X'])] print('reference pixel: {}'.format(inps.ref_yx)) # Coordinate system: geocoded or not inps.geocoded = False if 'Y_FIRST' in atr.keys(): inps.geocoded = True print('geocoded: {}'.format(inps.geocoded)) # Prepare DEM, inc_angle, lat/lon file for PyAPS to read if inps.geom_file: geom_obj = geometry(inps.geom_file) geom_obj.open() print('converting DEM/incAngle for PyAPS to read') # DEM dem = readfile.read(inps.geom_file, datasetName='height', print_msg=False)[0] inps.dem_file = 'pyapsDem.hgt' writefile.write(dem, inps.dem_file, metadata=atr) # inc_angle if 'incidenceAngle' in geom_obj.datasetNames: inps.inc_angle = readfile.read(inps.geom_file, datasetName='incidenceAngle', print_msg=False)[0] else: atr = readfile.read_attribute(inps.timeseries_file) inps.inc_angle = ut.incidence_angle(atr, dem=dem, dimension=0) inps.inc_angle = np.ones(dem.shape, dtype=np.float32) * inps.inc_angle inps.inc_angle_file = 'pyapsIncAngle.flt' writefile.write(inps.inc_angle, inps.inc_angle_file, metadata=atr) # latitude if 'latitude' in geom_obj.datasetNames: data = readfile.read(inps.geom_file, datasetName='latitude', print_msg=False)[0] print('converting lat for PyAPS to read') inps.lat_file = 'pyapsLat.flt' writefile.write(data, inps.lat_file, metadata=atr) else: inps.lat_file = None # longitude if 'longitude' in geom_obj.datasetNames: data = readfile.read(inps.geom_file, datasetName='longitude', print_msg=False)[0] print('converting lon for PyAPS to read') inps.lon_file = 'pyapsLon.flt' writefile.write(data, inps.lon_file, metadata=atr) else: inps.lon_file = None return inps, atr
def correct_dem_error(inps): """Correct DEM error of input timeseries file""" start_time = time.time() # limit the number of threads to 1 # for slight speedup and big CPU usage save num_threads_dict = cluster.set_num_threads("1") ## 1. input info # 1.1 read date info ts_obj = timeseries(inps.timeseries_file) ts_obj.open() num_date = ts_obj.numDate length, width = ts_obj.length, ts_obj.width num_step = len(inps.stepFuncDate) # exclude dates date_flag = read_exclude_date(inps.excludeDate, ts_obj.dateList)[0] if inps.polyOrder > np.sum(date_flag): raise ValueError( "input poly order {} > number of acquisition {}! Reduce it!". format(inps.polyOrder, np.sum(date_flag))) # 1.2 design matrix part 1 - time func for surface deformation G_defo = get_design_matrix4defo(inps) ## 2. prepare output # 2.1 metadata meta = dict(ts_obj.metadata) print( 'add/update the following configuration metadata to file:\n{}'.format( configKeys)) for key in configKeys: meta[key_prefix + key] = str(vars(inps)[key]) # 2.2 instantiate est. DEM error dem_err_file = 'demErr.h5' meta['FILE_TYPE'] = 'dem' meta['UNIT'] = 'm' ds_name_dict = {'dem': [np.float32, (length, width), None]} writefile.layout_hdf5(dem_err_file, ds_name_dict, metadata=meta) # 2.3 instantiate corrected time-series ts_cor_file = inps.outfile meta['FILE_TYPE'] = 'timeseries' writefile.layout_hdf5(ts_cor_file, metadata=meta, ref_file=inps.timeseries_file) # 2.4 instantiate residual phase time-series ts_res_file = os.path.join(os.path.dirname(inps.outfile), 'timeseriesResidual.h5') writefile.layout_hdf5(ts_res_file, metadata=meta, ref_file=inps.timeseries_file) ## 3. run the estimation and write to disk # 3.1 split ts_file into blocks to save memory # 1st dimension size: ts (obs / cor / res / step) + dem_err/inc_angle/rg_dist (+pbase) num_epoch = num_date * 3 + num_step + 3 if inps.geom_file: geom_obj = geometry(inps.geom_file) geom_obj.open(print_msg=False) if 'bperp' in geom_obj.datasetNames: num_epoch += num_date # split in row/line direction based on the input memory limit num_box = int( np.ceil((num_epoch * length * width * 4) * 2.5 / (inps.maxMemory * 1024**3))) box_list = cluster.split_box2sub_boxes(box=(0, 0, width, length), num_split=num_box, dimension='y') # 3.2 prepare the input arguments for *_patch() data_kwargs = { 'G_defo': G_defo, 'ts_file': inps.timeseries_file, 'geom_file': inps.geom_file, 'date_flag': date_flag, 'phase_velocity': inps.phaseVelocity, } # 3.3 invert / write block-by-block for i, box in enumerate(box_list): box_wid = box[2] - box[0] box_len = box[3] - box[1] if num_box > 1: print('\n------- processing patch {} out of {} --------------'. format(i + 1, num_box)) print('box width: {}'.format(box_wid)) print('box length: {}'.format(box_len)) # update box argument in the input data data_kwargs['box'] = box # invert if not inps.cluster: # non-parallel delta_z, ts_cor, ts_res = correct_dem_error_patch( **data_kwargs)[:-1] else: # parallel print('\n\n------- start parallel processing using Dask -------') # initiate the output data delta_z = np.zeros((box_len, box_wid), dtype=np.float32) ts_cor = np.zeros((num_date, box_len, box_wid), dtype=np.float32) ts_res = np.zeros((num_date, box_len, box_wid), dtype=np.float32) # initiate dask cluster and client cluster_obj = cluster.DaskCluster(inps.cluster, inps.numWorker, config_name=inps.config) cluster_obj.open() # run dask delta_z, ts_cor, ts_res = cluster_obj.run( func=correct_dem_error_patch, func_data=data_kwargs, results=[delta_z, ts_cor, ts_res]) # close dask cluster and client cluster_obj.close() print('------- finished parallel processing -------\n\n') # write the block to disk # with 3D block in [z0, z1, y0, y1, x0, x1] # and 2D block in [y0, y1, x0, x1] # DEM error - 2D block = [box[1], box[3], box[0], box[2]] writefile.write_hdf5_block(dem_err_file, data=delta_z, datasetName='dem', block=block) # corrected time-series - 3D block = [0, num_date, box[1], box[3], box[0], box[2]] writefile.write_hdf5_block(ts_cor_file, data=ts_cor, datasetName='timeseries', block=block) # residual time-series - 3D block = [0, num_date, box[1], box[3], box[0], box[2]] writefile.write_hdf5_block(ts_res_file, data=ts_res, datasetName='timeseries', block=block) # roll back to the origial number of threads cluster.roll_back_num_threads(num_threads_dict) # time info m, s = divmod(time.time() - start_time, 60) print('time used: {:02.0f} mins {:02.1f} secs.'.format(m, s)) return dem_err_file, ts_cor_file, ts_res_file
def write2hdf5(out_file, ts_file, coh_file, mask_file, geom_file, metadata): """Write HDF5 file in HDF-EOS5 format""" ts_obj = timeseries(ts_file) ts_obj.open(print_msg=False) dateList = ts_obj.dateList # Open HDF5 File f = h5py.File(out_file, 'w') print('create HDF5 file: {} with w mode'.format(out_file)) maxDigit = 20 # Write Observation - Displacement gName = 'HDFEOS/GRIDS/timeseries/observation' print('create group /{}'.format(gName)) group = f.create_group(gName) dsName = 'displacement' data = ts_obj.read(print_msg=False) print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}' ' with compression={c}').format(g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape, c=compression)) dset = group.create_dataset(dsName, data=data, dtype=np.float32, chunks=True, compression=compression) dset.attrs['Title'] = dsName dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = 'meters' dsName = 'date' data = np.array(dateList, dtype=np.string_) group.create_dataset(dsName, data=data) print('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'.format(g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape)) dsName = 'bperp' data = np.array(ts_obj.pbase, dtype=np.float32) group.create_dataset(dsName, data=data) print('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'.format(g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape)) # Write Quality gName = 'HDFEOS/GRIDS/timeseries/quality' print('create group /{}'.format(gName)) group = f.create_group(gName) ## 1 - temporalCoherence dsName = 'temporalCoherence' data = readfile.read(coh_file)[0] print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}' ' with compression={c}').format(g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape, c=compression)) dset = group.create_dataset(dsName, data=data, chunks=True, compression=compression) dset.attrs['Title'] = dsName dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = '1' ## 2 - mask dsName = 'mask' data = readfile.read(mask_file, datasetName='mask')[0] print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}' ' with compression={c}').format(g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape, c=compression)) dset = group.create_dataset(dsName, data=data, chunks=True, compression=compression) dset.attrs['Title'] = dsName dset.attrs['MissingValue'] = BOOL_ZERO dset.attrs['_FillValue'] = BOOL_ZERO dset.attrs['Units'] = '1' # Write Geometry # Required: height, incidenceAngle # Optional: rangeCoord, azimuthCoord, azimuthAngle, slantRangeDistance, waterMask, shadowMask gName = 'HDFEOS/GRIDS/timeseries/geometry' print('create group /{}'.format(gName)) group = f.create_group(gName) geom_obj = geometry(geom_file) geom_obj.open(print_msg=False) for dsName in geom_obj.datasetNames: data = geom_obj.read(datasetName=dsName, print_msg=False) print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}' ' with compression={c}').format(g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape, c=compression)) dset = group.create_dataset(dsName, data=data, chunks=True, compression=compression) dset.attrs['Title'] = dsName if dsName in ['height', 'slantRangeDistance', 'bperp']: dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = 'meters' elif dsName in ['incidenceAngle', 'azimuthAngle', 'latitude', 'longitude']: dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = 'degrees' elif dsName in ['rangeCoord', 'azimuthCoord']: dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = '1' elif dsName in ['waterMask', 'shadowMask']: dset.attrs['MissingValue'] = BOOL_ZERO dset.attrs['_FillValue'] = BOOL_ZERO dset.attrs['Units'] = '1' # Write Attributes to the HDF File print('write metadata to root level') for key, value in iter(metadata.items()): f.attrs[key] = value f.close() print('finished writing to {}'.format(out_file)) return out_file
def write_hdf5_file(metadata, out_file, ts_file, tcoh_file, scoh_file, mask_file, geom_file): """Write HDF5 file in HDF-EOS5 format.""" ts_obj = timeseries(ts_file) ts_obj.open(print_msg=False) dateList = ts_obj.dateList numDate = len(dateList) # Open HDF5 File f = h5py.File(out_file, 'w') print('create HDF5 file: {} with w mode'.format(out_file)) max_digit = 55 ##### Group - Observation gName = 'HDFEOS/GRIDS/timeseries/observation' print('create group /{}'.format(gName)) group = f.create_group(gName) ## O1 - displacement dsName = 'displacement' dsShape = (numDate, ts_obj.length, ts_obj.width) dsDataType = np.float32 print(('create dataset /{d:<{w}} of {t:<10} in size of {s}' ' with compression={c}').format(d='{}/{}'.format(gName, dsName), w=max_digit, t='float32', s=dsShape, c=COMPRESSION)) dset = group.create_dataset(dsName, shape=dsShape, maxshape=(None, dsShape[1], dsShape[2]), dtype=dsDataType, chunks=True, compression=COMPRESSION) print('write data acquition by acquition ...') prog_bar = ptime.progressBar(maxValue=numDate) for i in range(numDate): dset[i, :, :] = readfile.read(ts_file, datasetName=dateList[i])[0] prog_bar.update(i + 1, suffix='{}/{} {}'.format(i + 1, numDate, dateList[i])) prog_bar.close() # attributes dset.attrs['Title'] = dsName dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = 'meters' ## O2 - date dsName = 'date' data = np.array(dateList, dtype=np.string_) dset = create_hdf5_dataset(group, dsName, data) ## O3 - perp baseline dsName = 'bperp' data = np.array(ts_obj.pbase, dtype=np.float32) dset = create_hdf5_dataset(group, dsName, data) ##### Group - Quality gName = 'HDFEOS/GRIDS/timeseries/quality' print('create group /{}'.format(gName)) group = f.create_group(gName) ## Q1 - temporalCoherence dsName = 'temporalCoherence' # read data = readfile.read(tcoh_file)[0] # write dset = create_hdf5_dataset(group, dsName, data) # attributes dset.attrs['Title'] = dsName dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = '1' ## Q2 - avgSpatialCoherence dsName = 'avgSpatialCoherence' # read data = readfile.read(scoh_file)[0] # write dset = create_hdf5_dataset(group, dsName, data) # attributes dset.attrs['Title'] = dsName dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = '1' ## Q3 - mask dsName = 'mask' # read data = readfile.read(mask_file, datasetName='mask')[0] # write dset = create_hdf5_dataset(group, dsName, data) # attributes dset.attrs['Title'] = dsName dset.attrs['MissingValue'] = BOOL_ZERO dset.attrs['_FillValue'] = BOOL_ZERO dset.attrs['Units'] = '1' ##### Group - Write Geometry # Required: height, incidenceAngle # Optional: rangeCoord, azimuthCoord, azimuthAngle, slantRangeDistance, waterMask, shadowMask gName = 'HDFEOS/GRIDS/timeseries/geometry' print('create group /{}'.format(gName)) group = f.create_group(gName) geom_obj = geometry(geom_file) geom_obj.open(print_msg=False) for dsName in geom_obj.datasetNames: # read data = geom_obj.read(datasetName=dsName, print_msg=False) # write dset = create_hdf5_dataset(group, dsName, data) # attributes dset.attrs['Title'] = dsName if dsName in ['height', 'slantRangeDistance', 'bperp']: dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = 'meters' elif dsName in [ 'incidenceAngle', 'azimuthAngle', 'latitude', 'longitude' ]: dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = 'degrees' elif dsName in ['rangeCoord', 'azimuthCoord']: dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = '1' elif dsName in ['waterMask', 'shadowMask']: dset.attrs['MissingValue'] = BOOL_ZERO dset.attrs['_FillValue'] = BOOL_ZERO dset.attrs['Units'] = '1' # Write Attributes to the HDF File print('write metadata to root level') for key, value in iter(metadata.items()): f.attrs[key] = value f.close() print('finished writing to {}'.format(out_file)) return out_file
def calc_delay_timeseries(inps): """Calculate delay time-series and write it to HDF5 file. Parameters: inps : namespace, all input parameters Returns: tropo_file : str, file name of ECMWF.h5 """ def get_dataset_size(fname): atr = readfile.read_attribute(fname) shape = (int(atr['LENGTH']), int(atr['WIDTH'])) return shape def run_or_skip(grib_files, tropo_file, geom_file): print('update mode: ON') print('output file: {}'.format(tropo_file)) flag = 'skip' # check existance and modification time if ut.run_or_skip(out_file=tropo_file, in_file=grib_files, print_msg=False) == 'run': flag = 'run' print( '1) output file either do NOT exist or is NOT newer than all GRIB files.' ) else: print('1) output file exists and is newer than all GRIB files.') # check dataset size in space / time date_list = [ str(re.findall('\d{8}', os.path.basename(i))[0]) for i in grib_files ] if (get_dataset_size(tropo_file) != get_dataset_size(geom_file) or any(i not in timeseries(tropo_file).get_date_list() for i in date_list)): flag = 'run' print( '2) output file does NOT have the same len/wid as the geometry file {} or does NOT contain all dates' .format(geom_file)) else: print( '2) output file has the same len/wid as the geometry file and contains all dates' ) # check if output file is fully written with h5py.File(tropo_file, 'r') as f: if np.all(f['timeseries'][-1, :, :] == 0): flag = 'run' print('3) output file is NOT fully written.') else: print('3) output file is fully written.') # result print('run or skip: {}'.format(flag)) return flag if run_or_skip(inps.grib_files, inps.tropo_file, inps.geom_file) == 'skip': return ## 1. prepare geometry data geom_obj = geometry(inps.geom_file) geom_obj.open() inps.inc = geom_obj.read(datasetName='incidenceAngle') inps.dem = geom_obj.read(datasetName='height') # for testing if inps.custom_height: print( 'use input custom height of {} m for vertical integration'.format( inps.custom_height)) inps.dem[:] = inps.custom_height if 'latitude' in geom_obj.datasetNames: # for lookup table in radar-coord (isce, doris) inps.lat = geom_obj.read(datasetName='latitude') inps.lon = geom_obj.read(datasetName='longitude') elif 'Y_FIRST' in geom_obj.metadata: # for lookup table in geo-coded (gamma, roipac) and obs. in geo-coord inps.lat, inps.lon = ut.get_lat_lon(geom_obj.metadata) # convert coordinates to lat/lon, e.g. from UTM for ASF HyPP3 if not geom_obj.metadata['Y_UNIT'].startswith('deg'): inps.lat, inps.lon = ut.to_latlon(inps.atr['OG_FILE_PATH'], inps.lon, inps.lat) else: # for lookup table in geo-coded (gamma, roipac) and obs. in radar-coord inps.lat, inps.lon = ut.get_lat_lon_rdc(inps.atr) # mask of valid pixels mask = np.multiply(inps.inc != 0, ~np.isnan(inps.inc)) ## 2. prepare output file # metadata atr = inps.atr.copy() atr['FILE_TYPE'] = 'timeseries' atr['UNIT'] = 'm' # remove metadata related with double reference # because absolute delay is calculated and saved for key in ['REF_DATE', 'REF_X', 'REF_Y', 'REF_LAT', 'REF_LON']: if key in atr.keys(): atr.pop(key) # instantiate time-series length, width = int(atr['LENGTH']), int(atr['WIDTH']) num_date = len(inps.grib_files) date_list = [ str(re.findall('\d{8}', os.path.basename(i))[0]) for i in inps.grib_files ] dates = np.array(date_list, dtype=np.string_) ds_name_dict = { "date": [dates.dtype, (num_date, ), dates], "timeseries": [np.float32, (num_date, length, width), None], } writefile.layout_hdf5(inps.tropo_file, ds_name_dict, metadata=atr) ## 3. calculate phase delay print( '\n------------------------------------------------------------------------------' ) print( 'calculating absolute delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...' ) print('number of grib files used: {}'.format(num_date)) prog_bar = ptime.progressBar(maxValue=num_date, print_msg=~inps.verbose) for i in range(num_date): grib_file = inps.grib_files[i] # calc tropo delay tropo_data = get_delay(grib_file, tropo_model=inps.tropo_model, delay_type=inps.delay_type, dem=inps.dem, inc=inps.inc, lat=inps.lat, lon=inps.lon, mask=mask, verbose=inps.verbose) # write tropo delay to file block = [i, i + 1, 0, length, 0, width] writefile.write_hdf5_block(inps.tropo_file, data=tropo_data, datasetName='timeseries', block=block, print_msg=False) prog_bar.update(i + 1, suffix=os.path.basename(grib_file)) prog_bar.close() return inps.tropo_file
def calculate_delay_timeseries(inps): """Calculate delay time-series and write it to HDF5 file. Parameters: inps : namespace, all input parameters Returns: tropo_file : str, file name of ECMWF.h5 """ def get_dataset_size(fname): atr = readfile.read_attribute(fname) shape = (int(atr['LENGTH']), int(atr['WIDTH'])) return shape # check existing tropo delay file if (ut.run_or_skip(out_file=inps.tropo_file, in_file=inps.grib_files, print_msg=False) == 'skip' and get_dataset_size(inps.tropo_file) == get_dataset_size( inps.geom_file)): print( '{} file exists and is newer than all GRIB files, skip updating.'. format(inps.tropo_file)) return # prepare geometry data geom_obj = geometry(inps.geom_file) geom_obj.open() inps.dem = geom_obj.read(datasetName='height') inps.inc = geom_obj.read(datasetName='incidenceAngle') if 'latitude' in geom_obj.datasetNames: # for dataset in geo OR radar coord with lookup table in radar-coord (isce, doris) inps.lat = geom_obj.read(datasetName='latitude') inps.lon = geom_obj.read(datasetName='longitude') elif 'Y_FIRST' in geom_obj.metadata: # for geo-coded dataset (gamma, roipac) inps.lat, inps.lon = ut.get_lat_lon(geom_obj.metadata) else: # for radar-coded dataset (gamma, roipac) inps.lat, inps.lon = ut.get_lat_lon_rdc(geom_obj.metadata) # calculate phase delay length, width = int(inps.atr['LENGTH']), int(inps.atr['WIDTH']) num_date = len(inps.grib_files) date_list = [str(re.findall('\d{8}', i)[0]) for i in inps.grib_files] tropo_data = np.zeros((num_date, length, width), np.float32) print( '\n------------------------------------------------------------------------------' ) print( 'calcualting absolute delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...' ) print('number of grib files used: {}'.format(num_date)) prog_bar = ptime.progressBar(maxValue=num_date) for i in range(num_date): grib_file = inps.grib_files[i] tropo_data[i] = get_delay(grib_file, inps) prog_bar.update(i + 1, suffix=os.path.basename(grib_file)) prog_bar.close() # remove metadata related with double reference # because absolute delay is calculated and saved for key in ['REF_DATE', 'REF_X', 'REF_Y', 'REF_LAT', 'REF_LON']: if key in inps.atr.keys(): inps.atr.pop(key) # Write tropospheric delay to HDF5 ts_obj = timeseries(inps.tropo_file) ts_obj.write2hdf5(data=tropo_data, dates=date_list, metadata=inps.atr, refFile=inps.timeseries_file) return inps.tropo_file
def check_loaded_dataset(work_dir='./', print_msg=True, relpath=False): """Check the result of loading data for the following two rules: 1. file existance 2. file attribute readability Parameters: work_dir : string, MintPy working directory print_msg : bool, print out message Returns: True, if all required files and dataset exist; otherwise, ERROR If True, PROCESS, SLC folder could be removed. stack_file : geom_file : lookup_file : Example: work_dir = os.path.expandvars('./FernandinaSenDT128/mintpy') ut.check_loaded_dataset(work_dir) """ load_complete = True if not work_dir: work_dir = os.getcwd() work_dir = os.path.abspath(work_dir) # 1. interferograms stack file: unwrapPhase, coherence ds_list = ['unwrapPhase', 'rangeOffset', 'azimuthOffset'] flist = [os.path.join(work_dir, 'inputs/ifgramStack.h5')] stack_file = is_file_exist(flist, abspath=True) if stack_file is not None: obj = ifgramStack(stack_file) obj.open(print_msg=False) if all(x not in obj.datasetNames for x in ds_list): msg = 'required dataset is missing in file {}:'.format(stack_file) msg += '\n' + ' OR '.join(ds_list) raise ValueError(msg) # check coherence for phase stack if 'unwrapPhase' in obj.datasetNames and 'coherence' not in obj.datasetNames: print('WARNING: "coherence" is missing in file {}'.format( stack_file)) else: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), './inputs/ifgramStack.h5') atr = readfile.read_attribute(stack_file) # 2. geom_file: height if 'X_FIRST' in atr.keys(): flist = [os.path.join(work_dir, 'inputs/geometryGeo.h5')] else: flist = [os.path.join(work_dir, 'inputs/geometryRadar.h5')] geom_file = is_file_exist(flist, abspath=True) if geom_file is not None: obj = geometry(geom_file) obj.open(print_msg=False) dname = geometryDatasetNames[0] if dname not in obj.datasetNames: raise ValueError( 'required dataset "{}" is missing in file {}'.format( dname, geom_file)) else: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), './inputs/geometry*.h5') # 3. lookup_file: latitude,longitude or rangeCoord,azimuthCoord # could be different than geometry file in case of roipac and gamma flist = [os.path.join(work_dir, 'inputs/geometry*.h5')] lookup_file = get_lookup_file(flist, abspath=True, print_msg=print_msg) if 'X_FIRST' not in atr.keys(): if lookup_file is not None: obj = geometry(lookup_file) obj.open(print_msg=False) if atr['PROCESSOR'] in ['isce', 'doris']: dnames = geometryDatasetNames[1:3] elif atr['PROCESSOR'] in ['gamma', 'roipac']: dnames = geometryDatasetNames[3:5] else: raise AttributeError('InSAR processor: {}'.format( atr['PROCESSOR'])) for dname in dnames: if dname not in obj.datasetNames: load_complete = False raise Exception( 'required dataset "{}" is missing in file {}'.format( dname, lookup_file)) else: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), './inputs/geometry*.h5') else: print("Input data seems to be geocoded. Lookup file not needed.") if relpath: stack_file = os.path.relpath(stack_file) if stack_file else stack_file geom_file = os.path.relpath(geom_file) if geom_file else geom_file lookup_file = os.path.relpath( lookup_file) if lookup_file else lookup_file # print message if print_msg: print(('Loaded dataset are processed by ' 'InSAR software: {}'.format(atr['PROCESSOR']))) if 'X_FIRST' in atr.keys(): print('Loaded dataset is in GEO coordinates') else: print('Loaded dataset is in RADAR coordinates') print('Interferograms Stack: {}'.format(stack_file)) print('Geometry File : {}'.format(geom_file)) print('Lookup Table File : {}'.format(lookup_file)) if load_complete: print('-' * 50) print( 'All data needed found/loaded/copied. Processed 2-pass InSAR data can be removed.' ) print('-' * 50) return load_complete, stack_file, geom_file, lookup_file
def write2hdf5(out_file, ts_file, coh_file, mask_file, geom_file, metadata): """Write HDF5 file in HDF-EOS5 format""" ts_obj = timeseries(ts_file) ts_obj.open(print_msg=False) dateList = ts_obj.dateList # Open HDF5 File f = h5py.File(out_file, 'w') print('create HDF5 file: {} with w mode'.format(out_file)) maxDigit = 20 # Write Observation - Displacement gName = 'HDFEOS/GRIDS/timeseries/observation' print('create group /{}'.format(gName)) group = f.create_group(gName) dsName = 'displacement' data = ts_obj.read(print_msg=False) print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}' ' with compression={c}').format(g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape, c=compression)) dset = group.create_dataset(dsName, data=data, dtype=np.float32, chunks=True, compression=compression) dset.attrs['Title'] = dsName dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = 'meters' dsName = 'date' data = np.array(dateList, dtype=np.string_) group.create_dataset(dsName, data=data) print('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'.format( g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape)) dsName = 'bperp' data = np.array(ts_obj.pbase, dtype=np.float32) group.create_dataset(dsName, data=data) print('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}'.format( g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape)) # Write Quality gName = 'HDFEOS/GRIDS/timeseries/quality' print('create group /{}'.format(gName)) group = f.create_group(gName) ## 1 - temporalCoherence dsName = 'temporalCoherence' data = readfile.read(coh_file)[0] print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}' ' with compression={c}').format(g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape, c=compression)) dset = group.create_dataset(dsName, data=data, chunks=True, compression=compression) dset.attrs['Title'] = dsName dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = '1' ## 2 - mask dsName = 'mask' data = readfile.read(mask_file, datasetName='mask')[0] print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}' ' with compression={c}').format(g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape, c=compression)) dset = group.create_dataset(dsName, data=data, chunks=True, compression=compression) dset.attrs['Title'] = dsName dset.attrs['MissingValue'] = BOOL_ZERO dset.attrs['_FillValue'] = BOOL_ZERO dset.attrs['Units'] = '1' # Write Geometry # Required: height, incidenceAngle # Optional: rangeCoord, azimuthCoord, azimuthAngle, slantRangeDistance, waterMask, shadowMask gName = 'HDFEOS/GRIDS/timeseries/geometry' print('create group /{}'.format(gName)) group = f.create_group(gName) geom_obj = geometry(geom_file) geom_obj.open(print_msg=False) for dsName in geom_obj.datasetNames: data = geom_obj.read(datasetName=dsName, print_msg=False) print(('create dataset /{g}/{d:<{w}} of {t:<10} in size of {s}' ' with compression={c}').format(g=gName, d=dsName, w=maxDigit, t=str(data.dtype), s=data.shape, c=compression)) dset = group.create_dataset(dsName, data=data, chunks=True, compression=compression) dset.attrs['Title'] = dsName if dsName in ['height', 'slantRangeDistance', 'bperp']: dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = 'meters' elif dsName in [ 'incidenceAngle', 'azimuthAngle', 'latitude', 'longitude' ]: dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = 'degrees' elif dsName in ['rangeCoord', 'azimuthCoord']: dset.attrs['MissingValue'] = FLOAT_ZERO dset.attrs['_FillValue'] = FLOAT_ZERO dset.attrs['Units'] = '1' elif dsName in ['waterMask', 'shadowMask']: dset.attrs['MissingValue'] = BOOL_ZERO dset.attrs['_FillValue'] = BOOL_ZERO dset.attrs['Units'] = '1' # Write Attributes to the HDF File print('write metadata to root level') for key, value in iter(metadata.items()): f.attrs[key] = value f.close() print('finished writing to {}'.format(out_file)) return out_file