def get_delay_radar(ztd_file, cos_inc_angle, pts_new): """calc single path tropo delay in line-of-sight direction Parameters: ztd_file - str, path of zenith delay file cos_inc_angle - 2D np.ndarray in (len, wid) in float32, cos(inc_angle) pts_new - 2D np.ndarray in (len*wid, 2) in float32 Returns: delay - 2D np.ndarray in float32, LOS delay """ # read ztd file delay_ztd, atr_ztd = readfile.read(ztd_file) # flip to be consistent with the reversed lats delay_ztd = np.flipud(delay_ztd) # pixel coordinates in ztd file lats, lons = ut.get_lat_lon(atr_ztd, dimension=1) # set lats in ascending order as required by RGI lats = np.flipud(lats) pts_ztd = ((lats.flatten(), lons.flatten())) # resample in pts_new coordinates RGI_func = RGI(pts_ztd, delay_ztd, method='nearest', bounds_error=False, fill_value=0) delay = RGI_func(pts_new) delay = delay.reshape(cos_inc_angle.shape) # project from zenith to line-of-sight delay /= cos_inc_angle # reverse the sign for consistency between different phase correction steps/methods delay *= -1 return delay
def flatten_lat_lon(box, ts_obj, coords=None): if coords is None: lats, lons = ut.get_lat_lon(ts_obj.metadata, box=box) lats = sorted(lats.flatten()) lons = sorted(lons.flatten()) return lats, lons
def ll2xy(inps): """transfer lat/lon to local coordination""" inps.metadata = readfile.read_attribute(inps.file[0]) # read geometry inps.lat, inps.lon = ut.get_lat_lon(inps.metadata) #inps.inc_angle = readfile.read(inps.geometry[0], datasetName='incidenceAngle')[0] #inps.head_angle = np.ones(inps.inc_angle.shape, dtype=np.float32) * float(inps.metadata['HEADING']) #inps.height = readfile.read(inps.geometry[0], datasetName='height')[0] # read mask file inps.mask = readfile.read(inps.file[0])[0] # mask data #inps.lat[inps.mask==0] = np.nan #inps.lon[inps.mask==0] = np.nan #inps.inc_angle[inps.mask==0] = np.nan #inps.head_angle[inps.mask==0] = np.nan #inps.height[inps.mask==0] = np.nan # conver latlon to xy # origin point origin_lat = (inps.lat[0, 0] + inps.lat[-1, 0]) / 2 origin_lon = (inps.lon[0, 0] + inps.lon[0, -1]) / 2 lat = np.transpose(inps.lat.reshape(-1, 1)) lon = np.transpose(inps.lon.reshape(-1, 1)) llh = np.vstack((lon, lat)) origin = np.array([origin_lon, origin_lat], dtype=float) XY = np.transpose( mut.llh2xy(llh, origin) ) * 1000 # unit of X/Y is meter and is a [N,2] matrix with [N,0] is X; [N,1] is Y X = XY[:, 0] Y = XY[:, 1] return X, Y, origin
def calculate_delay_timeseries(inps): """Calculate delay time-series and write it to HDF5 file. Parameters: inps : namespace, all input parameters Returns: tropo_file : str, file name of ECMWF.h5 """ def get_dataset_size(fname): atr = readfile.read_attribute(fname) shape = (int(atr['LENGTH']), int(atr['WIDTH'])) return shape # check existing tropo delay file if (ut.run_or_skip(out_file=inps.tropo_file, in_file=inps.grib_files, print_msg=False) == 'skip' and get_dataset_size(inps.tropo_file) == get_dataset_size( inps.geom_file)): print( '{} file exists and is newer than all GRIB files, skip updating.'. format(inps.tropo_file)) return # prepare geometry data geom_obj = geometry(inps.geom_file) geom_obj.open() inps.dem = geom_obj.read(datasetName='height') inps.inc = geom_obj.read(datasetName='incidenceAngle') if 'latitude' in geom_obj.datasetNames: # for dataset in geo OR radar coord with lookup table in radar-coord (isce, doris) inps.lat = geom_obj.read(datasetName='latitude') inps.lon = geom_obj.read(datasetName='longitude') elif 'Y_FIRST' in geom_obj.metadata: # for geo-coded dataset (gamma, roipac) inps.lat, inps.lon = ut.get_lat_lon(geom_obj.metadata) else: # for radar-coded dataset (gamma, roipac) inps.lat, inps.lon = ut.get_lat_lon_rdc(geom_obj.metadata) # calculate phase delay length, width = int(inps.atr['LENGTH']), int(inps.atr['WIDTH']) num_date = len(inps.grib_files) date_list = [str(re.findall('\d{8}', i)[0]) for i in inps.grib_files] tropo_data = np.zeros((num_date, length, width), np.float32) print( '\n------------------------------------------------------------------------------' ) print( 'calcualting absolute delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...' ) print('number of grib files used: {}'.format(num_date)) prog_bar = ptime.progressBar(maxValue=num_date) for i in range(num_date): grib_file = inps.grib_files[i] tropo_data[i] = get_delay(grib_file, inps) prog_bar.update(i + 1, suffix=os.path.basename(grib_file)) prog_bar.close() # remove metadata related with double reference # because absolute delay is calculated and saved for key in ['REF_DATE', 'REF_X', 'REF_Y', 'REF_LAT', 'REF_LON']: if key in inps.atr.keys(): inps.atr.pop(key) # Write tropospheric delay to HDF5 ts_obj = timeseries(inps.tropo_file) ts_obj.write2hdf5(data=tropo_data, dates=date_list, metadata=inps.atr, refFile=inps.timeseries_file) return inps.tropo_file
def main(iargs=None): inps = cmd_line_parse(iargs) inps.work_dir = os.path.abspath(os.path.dirname(inps.ts_file)) inps.cbar_file = os.path.join(inps.work_dir, 'google_earth_cbar.png') inps.star_file = os.path.join(inps.work_dir, "star.png") inps.dot_file = os.path.join(inps.work_dir, "shaded_dot.png") inps.dygraph_file = os.path.join(inps.work_dir, "dygraph-combined.js") inps.kml_data_dir = os.path.join(inps.work_dir, 'kml_data') ## Define file names if inps.outfile: inps.outfile_base = os.path.splitext(os.path.basename(inps.outfile))[0] else: inps.outfile_base = plot.auto_figure_title(inps.ts_file, inps_dict=vars(inps)) kml_root_file = os.path.join(inps.work_dir, '{}_root.kml'.format(inps.outfile_base)) kmz_file = os.path.join(inps.work_dir, '{}.kmz'.format(inps.outfile_base)) ## read data ts_obj = timeseries(inps.ts_file) ts_obj.open() length, width = ts_obj.length, ts_obj.width inps.metadata = ts_obj.metadata lats, lons = ut.get_lat_lon(ts_obj.metadata) print('input data shape in row/col: {}/{}'.format(length, width)) vel = readfile.read(inps.vel_file, datasetName='velocity')[0] * 100. # Set min/max velocity for colormap if inps.vlim is None: inps.vlim = [np.nanmin(vel), np.nanmax(vel)] if inps.wrap: print('re-wrapping data to {} cm/year for color coding'.format( inps.vlim)) ##--------- Create root KML file with network links to data KML files --------------## kml_root_doc = KML.Document() # 1 Create Overlay element for colorbar cbar_overlay = generate_cbar_element(cbar_file=inps.cbar_file, vmin=inps.vlim[0], vmax=inps.vlim[1], cmap=inps.colormap) kml_root_doc.append(cbar_overlay) # 2 Generate the placemark for the Reference Pixel ref_point = create_reference_point_element(inps, lats, lons, ts_obj) print('add reference point.') ref_folder = KML.Folder(KML.name("ReferencePoint")) ref_folder.append(ref_point) kml_root_doc.append(ref_folder) # 3 Create data folder to contain actual data elements data_folder = KML.Folder(KML.name("Data")) for i, step in enumerate(inps.steps): net_link = generate_network_link(inps, ts_obj, step=step, lod=(inps.lods[i], inps.lods[i + 1])) if net_link is not None: data_folder.append(net_link) kml_root_doc.append(data_folder) ##---------------------------- Write root KML file ------------------------------## print('-' * 30) print('writing ' + kml_root_file) kml_root = KML.kml() kml_root.append(kml_root_doc) with open(kml_root_file, 'w') as f: f.write(etree.tostring(kml_root, pretty_print=True).decode('utf-8')) ## Copy auxiliary files res_dir = os.path.join(os.path.dirname(mintpy.__file__), "data") for fname in [inps.star_file, inps.dot_file, inps.dygraph_file]: src_file = os.path.join(res_dir, os.path.basename(fname)) shutil.copy2(src_file, inps.work_dir) print("copy {} to the local directory".format(src_file)) ## Generate KMZ file # 1) go to the directory of kmz file run_dir = os.path.abspath(os.getcwd()) os.chdir(inps.work_dir) # 2) zip all data files with ZipFile(kmz_file, 'w') as fz: kml_data_files = get_all_file_paths(inps.kml_data_dir) for fname in [ kml_root_file, inps.cbar_file, inps.dygraph_file, inps.dot_file, inps.star_file ] + kml_data_files: fz.write(os.path.relpath(fname)) os.remove(fname) shutil.rmtree(inps.kml_data_dir) # 3) go back to the running directory os.chdir(run_dir) print('merged all files to {}'.format(kmz_file)) print('Done.') print('Open {} in Google Earth and play!'.format(kmz_file)) return
def create_kml_region_document(inps, box_list, ts_obj, step): """Create list of KML.Document() objects for one level of details defined by box_list and step """ dot_file = '../../{}'.format(os.path.basename(inps.dot_file)) ## 1. read data file into timeseries object region_docs = [] num_box = len(box_list) for i in range(num_box): box = box_list[i] if box is None: box = (0, 0, ts_obj.width, ts_obj.length) length = box[3] - box[1] width = box[2] - box[0] # 1.1 Parse Date dates = np.array( ts_obj.times ) # 1D np.array of dates in datetime.datetime object in size of [num_date,] dates = list(map(lambda d: d.strftime("%Y-%m-%d"), dates)) num_date = len(dates) # 1.2 Parse Spatial coordinates lats, lons = ut.get_lat_lon(ts_obj.metadata, box=box) rows, cols = np.mgrid[box[1]:box[3] - 1:length * 1j, box[0]:box[2] - 1:width * 1j] # 1.3 Read Velocity / time-series / temporal coherence data vel = readfile.read(inps.vel_file, datasetName='velocity', box=box)[0] * 100. vel_std = readfile.read( inps.vel_file, datasetName='velocityStd', box=box)[0] * 100. ts_data = readfile.read(inps.ts_file, box=box)[0] * 100. ts_data -= np.tile( ts_data[0, :, :], (ts_data.shape[0], 1, 1)) # enforce displacement starts from zero temp_coh = readfile.read(inps.tcoh_file, box=box)[0] mask = readfile.read(inps.mask_file, box=box)[0] vel_c = np.array(vel, dtype=np.float32) if inps.wrap: vel_c = inps.vlim[0] + np.mod(vel_c - inps.vlim[0], inps.vlim[1] - inps.vlim[0]) ## 2. Create KML Document kml_document = KML.Document() # 2.1 Set and normalize colormap to defined vlim colormap = mpl.cm.get_cmap(inps.colormap) norm = mpl.colors.Normalize(vmin=inps.vlim[0], vmax=inps.vlim[1]) # 2.2 Set number of pixels to use num_pixel = int(length / step) * int(width / step) msg = "create KML doc for box {}/{}: {}".format(i + 1, num_box, box) msg += ", step: {} pixels, {} pixels in total ...".format( step, num_pixel) print(msg) # 2.3 Create data folder for all points data_folder = KML.Folder(KML.name("Data")) for i in range(0, length, step): for j in range(0, width, step): if mask[i, j]: # add point if it's not marked as masked out lat = lats[i, j] lon = lons[i, j] row = rows[i, j] col = cols[i, j] ts = ts_data[:, i, j] v = vel[i, j] vc = vel_c[i, j] vstd = vel_std[i, j] tcoh = temp_coh[i, j] # 2.3.1 Create KML icon style element style = KML.Style( KML.IconStyle( KML.color(get_hex_color(vc, colormap, norm)), KML.scale(0.5), KML.Icon(KML.href("{}".format(dot_file))))) # 2.3.2 Create KML point element point = KML.Point(KML.coordinates("{},{}".format(lon, lat))) js_data_string = generate_js_datastring( dates, inps.dygraph_file, num_date, ts) # 2.3.3 Create KML description element stats_info = get_description_string((lat, lon), (row, col), v, vstd, ts[-1], tcoh=tcoh) description = KML.description(stats_info, js_data_string) # 2.3.4 Crate KML Placemark element to hold style, description, and point elements placemark = KML.Placemark(style, description, point) # 2.3.5 Append each placemark element to the KML document object data_folder.append(placemark) # 2.4 Append data folder to KML document kml_document.append(data_folder) # 2.5 Append KML document to list of regionalized documents region_docs.append(kml_document) return region_docs
def read_data(inps): """ Returns: defo: 2D np.array with in-valid/masked-out pixel in NaN """ # metadata inps.metadata = readfile.read_attribute(inps.file) k = inps.metadata['FILE_TYPE'] inps.range2phase = -4. * np.pi / float(inps.metadata['WAVELENGTH']) # mask if inps.mask_file: inps.mask = readfile.read(inps.mask_file)[0] else: inps.mask = np.ones((int(inps.metadata['LENGTH']), int(inps.metadata['WIDTH'])), dtype=np.bool_) # data if k in ['.unw','velocity']: inps.phase = readfile.read(inps.file)[0] if k == 'velocity': # velocity to displacement date1, date2 = inps.metadata['DATE12'].split('_') dt1, dt2 = ptime.date_list2vector([date1, date2])[0] inps.phase *= (dt2 - dt1).days / 365.25 # displacement to phase inps.phase *= inps.range2phase # update mask to exclude pixel with NaN value inps.mask *= ~np.isnan(inps.phase) # set all masked out pixel to NaN inps.phase[inps.mask==0] = np.nan else: raise ValueError("input file not support yet: {}".format(k)) print('number of pixels: {}'.format(np.sum(inps.mask))) # change reference point if inps.ref_lalo: coord = ut.coordinate(inps.metadata) ref_lat, ref_lon = inps.ref_lalo ref_y, ref_x = coord.geo2radar(ref_lat, ref_lon)[0:2] # update data inps.phase -= inps.phase[ref_y, ref_x] # update metadata inps.metadata['REF_LAT'] = ref_lat inps.metadata['REF_LON'] = ref_lon inps.metadata['REF_Y'] = ref_y inps.metadata['REF_X'] = ref_x # read geometry inps.lat, inps.lon = ut.get_lat_lon(inps.metadata) inps.inc_angle = readfile.read(inps.geom_file, datasetName='incidenceAngle')[0] inps.head_angle = np.ones(inps.inc_angle.shape, dtype=np.float32) * float(inps.metadata['HEADING']) inps.height = readfile.read(inps.geom_file, datasetName='height')[0] # convert the height of ellipsoid to geoid (mean sea level) # ref: https://github.com/vandry/geoidheight if inps.ellipsoid2geoid: # import geoid module try: import geoid except: raise ImportError('Can not import geoidheight!') # calculate offset and correct height egm_file = os.path.join(os.path.dirname(geoid.__file__), 'geoids/egm2008-1.pgm') gh_obj = geoid.GeoidHeight(egm_file) h_offset = gh_obj.get(lat=np.nanmean(inps.lat), lon=np.nanmean(inps.lon)) inps.height -= h_offset # print message msg = 'convert height from ellipsoid to geoid' msg += '\n\tby subtracting a constant offset of {:.2f} m'.format(h_offset) print(msg) inps.lat[inps.mask==0] = np.nan inps.lon[inps.mask==0] = np.nan inps.inc_angle[inps.mask==0] = np.nan inps.head_angle[inps.mask==0] = np.nan inps.height[inps.mask==0] = np.nan # output filename if not inps.outfile: proj_name = sensor.project_name2sensor_name(inps.file)[1] if not proj_name: raise ValueError('No custom/auto output filename found.') inps.outfile = '{}_{}.mat'.format(proj_name, inps.metadata['DATE12']) if not inps.outdir: inps.outdir = os.path.dirname(inps.file) inps.outfile = os.path.join(inps.outdir, inps.outfile) inps.outfile = os.path.abspath(inps.outfile) return
def get_delay_timeseries(inps, atr): """Calculate delay time-series and write it to HDF5 file. Parameters: inps : namespace, all input parameters atr : dict, metadata to be saved in trop_file Returns: trop_file : str, file name of ECMWF.h5 """ def get_dataset_size(fname): atr = readfile.read_attribute(fname) return (atr['LENGTH'], atr['WIDTH']) # check 1 - existing tropo delay file if (ut.run_or_skip(out_file=inps.trop_file, in_file=inps.grib_file_list, print_msg=False) == 'skip' and get_dataset_size(inps.trop_file) == get_dataset_size(inps.geom_file)): print('{} file exists and is newer than all GRIB files, skip updating.'.format(inps.trop_file)) return # check 2 - geometry file if any(i is None for i in [inps.geom_file, inps.ref_yx]): print('No DEM / incidenceAngle / ref_yx found, skip calculating tropospheric delays.') if not os.path.isfile(inps.trop_file): inps.trop_file = None return # prepare geometry data geom_obj = geometry(inps.geom_file) geom_obj.open() inps.dem = geom_obj.read(datasetName='height') inps.inc = geom_obj.read(datasetName='incidenceAngle') if 'latitude' in geom_obj.datasetNames: # for dataset in geo OR radar coord with lookup table in radar-coord (isce, doris) inps.lat = geom_obj.read(datasetName='latitude') inps.lon = geom_obj.read(datasetName='longitude') elif 'Y_FIRST' in geom_obj.metadata: # for geo-coded dataset (gamma, roipac) inps.lat, inps.lon = ut.get_lat_lon(geom_obj.metadata) else: # for radar-coded dataset (gamma, roipac) inps.lat, inps.lon = ut.get_lat_lon_rdc(geom_obj.metadata) # calculate phase delay length, width = int(atr['LENGTH']), int(atr['WIDTH']) num_date = len(inps.grib_file_list) date_list = [str(re.findall('\d{8}', i)[0]) for i in inps.grib_file_list] trop_data = np.zeros((num_date, length, width), np.float32) print('calcualting delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...') print('number of grib files used: {}'.format(num_date)) prog_bar = ptime.progressBar(maxValue=num_date) for i in range(num_date): grib_file = inps.grib_file_list[i] trop_data[i] = get_delay(grib_file, inps) prog_bar.update(i+1, suffix=os.path.basename(grib_file)) prog_bar.close() # Convert relative phase delay on reference date inps.ref_date = atr.get('REF_DATE', date_list[0]) print('convert to relative phase delay with reference date: '+inps.ref_date) inps.ref_idx = date_list.index(inps.ref_date) trop_data -= np.tile(trop_data[inps.ref_idx, :, :], (num_date, 1, 1)) # Write tropospheric delay to HDF5 atr['REF_Y'] = inps.ref_yx[0] atr['REF_X'] = inps.ref_yx[1] ts_obj = timeseries(inps.trop_file) ts_obj.write2hdf5(data=trop_data, dates=date_list, metadata=atr, refFile=inps.timeseries_file) return
def write_shape_file(fDict, shp_file, box=None): '''Write time-series data to a shape file Parameters: fDict - dict, with value for path of data files shp_file - str, output filename box - tuple of 4 int, in (x0, y0, x1, y1) Returns: shp_file - str, output filename ''' shpDriver = ogr.GetDriverByName("ESRI Shapefile") print('output shape file: {}'.format(shp_file)) ##Check if shape file already exists if os.path.exists(shp_file): print('output shape file: {} exists, will be overwritten ....'.format( shp_file)) shpDriver.DeleteDataSource(shp_file) ##Start creating shapefile dataset and layer definition ds = shpDriver.CreateDataSource(shp_file) srs = ogr.osr.SpatialReference() srs.ImportFromEPSG(4326) layer = ds.CreateLayer('mintpy', srs, geom_type=ogr.wkbPoint) #Add code for each point fd = ogr.FieldDefn('CODE', ogr.OFTString) fd.SetWidth(8) layer.CreateField(fd) #Add DEM height for each point - this could be before / after DEM error correction fd = ogr.FieldDefn('HEIGHT', ogr.OFTReal) fd.SetWidth(7) fd.SetPrecision(2) layer.CreateField(fd) #Supposed to represent DEM error estimation uncertainty fd = ogr.FieldDefn('H_STDEV', ogr.OFTReal) fd.SetWidth(5) fd.SetPrecision(2) layer.CreateField(fd) #Estimated LOS velocity fd = ogr.FieldDefn('VEL', ogr.OFTReal) fd.SetWidth(8) fd.SetPrecision(2) layer.CreateField(fd) #Estimated uncertainty in velocity fd = ogr.FieldDefn('V_STDEV', ogr.OFTReal) fd.SetWidth(6) fd.SetPrecision(2) layer.CreateField(fd) #Temporal coherence fd = ogr.FieldDefn('COHERENCE', ogr.OFTReal) fd.SetWidth(5) fd.SetPrecision(3) layer.CreateField(fd) #Effective area - SqueeSAR DS / PS layer.CreateField(ogr.FieldDefn('EFF_AREA', ogr.OFTInteger)) ##Time to load the dates from time-series HDF5 field and create one attribute for each date ts_obj = timeseries(fDict['TimeSeries']) ts_obj.open(print_msg=False) for date in ts_obj.dateList: fd = ogr.FieldDefn('D{0}'.format(date), ogr.OFTReal) fd.SetWidth(8) fd.SetPrecision(2) layer.CreateField(fd) layerDefn = layer.GetLayerDefn() ####Total number of points mask = readfile.read(fDict['Mask'], box=box)[0] nValid = np.sum(mask != 0) print('number of points with time-series:', nValid) lats, lons = ut.get_lat_lon(ts_obj.metadata, geom_file=fDict['Geometry'], box=box) ###Loop over all datasets in context managers to skip close statements with h5py.File(fDict['TimeSeries'], 'r') as tsid: with h5py.File(fDict['Coherence'], 'r') as cohid: with h5py.File(fDict['Velocity'], 'r') as velid: with h5py.File(fDict['Geometry'], 'r') as geomid: length = box[3] - box[1] width = box[2] - box[0] #Start counter counter = 1 prog_bar = ptime.progressBar(maxValue=nValid) #For each line for i in range(length): line = i + box[1] # read data for the line ts = tsid['timeseries'][:, line, box[0]:box[2]].astype( np.float64) coh = cohid['temporalCoherence'][line, box[0]:box[2]].astype( np.float64) vel = velid['velocity'][line, box[0]:box[2]].astype( np.float64) vel_std = velid['velocityStd'][line, box[0]:box[2]].astype( np.float64) hgt = geomid['height'][line, box[0]:box[2]].astype( np.float64) lat = lats[i, :].astype(np.float64) lon = lons[i, :].astype(np.float64) for j in range(width): if mask[i, j] == 0: continue #Create metadata dict rdict = { 'CODE': hex(counter)[2:].zfill(8), 'HEIGHT': hgt[j], 'H_STDEV': 0., 'VEL': vel[j] * 1000, 'V_STDEV': vel_std[j] * 1000, 'COHERENCE': coh[j], 'EFF_AREA': 1 } for ind, date in enumerate(ts_obj.dateList): rdict['D{0}'.format(date)] = ts[ind, j] * 1000 #Create feature with definition feature = ogr.Feature(layerDefn) add_metadata(feature, [lon[j], lat[j]], rdict) layer.CreateFeature(feature) feature = None # update counter / progress bar counter += 1 prog_bar.update(counter, every=100, suffix='{}/{}'.format( counter, nValid)) prog_bar.close() print('finished writing to file: {}'.format(shp_file)) return shp_file
def prep_gbis(inps): """prepare data that has to be written in *.mat file""" if str.find(str(inps.file), 'S1') != -1: key1 = 'S1' key2 = '.he5' for file in os.listdir(os.getcwd()): if str.find(file, key1) != -1 and str.find(file, key2) != -1: shutil.copy(file, inps.outdir) os.chdir("".join(inps.outdir)) geom_file = multitrack_utilities.find_HDFEOS_fullname(os.getcwd()) else: geom_file = 'geo_geometryRadar.h5' # metadata unw_file = 'geo_' + inps.startDate + '_' + inps.endDate + '.unw' inps.metadata = readfile.read_attribute(unw_file) inps.phase, atr = readfile.read(unw_file) # mask if not inps.mask_file or inps.mask_file == 'None': inps.mask = np.ones( (int(inps.metadata['LENGTH']), int(inps.metadata['WIDTH'])), dtype=np.bool_) else: inps.mask = readfile.read('geo_' + inps.mask_file)[0] # update mask to exclude pixel with NaN value inps.mask *= ~np.isnan(inps.phase) # set all masked out pixel to NaN inps.phase[inps.mask == 0] = np.nan # change reference point if inps.ref_lalo: coord = ut.coordinate(inps.metadata) ref_lat, ref_lon = inps.ref_lalo ref_y, ref_x = coord.geo2radar(ref_lat, ref_lon)[0:2] # update data inps.phase -= inps.phase[ref_y, ref_x] # update metadata inps.metadata['REF_LAT'] = ref_lat inps.metadata['REF_LON'] = ref_lon inps.metadata['REF_Y'] = ref_y inps.metadata['REF_X'] = ref_x # read geometry inps.lat, inps.lon = ut.get_lat_lon(inps.metadata) inps.inc_angle = readfile.read(geom_file, datasetName='incidenceAngle')[0] inps.head_angle = np.ones(inps.inc_angle.shape, dtype=np.float32) * float( inps.metadata['HEADING']) inps.height = readfile.read(geom_file, datasetName='height')[0] inps.lat[inps.mask == 0] = np.nan inps.lon[inps.mask == 0] = np.nan inps.inc_angle[inps.mask == 0] = np.nan inps.head_angle[inps.mask == 0] = np.nan inps.height[inps.mask == 0] = np.nan # output filename proj_name = atr['PROJECT_NAME'] if not proj_name: raise ValueError('No custom/auto output filename found.') inps.outfile = '{}_{}_{}.mat'.format(proj_name, inps.startDate, inps.endDate) inps.outfile = os.path.join(inps.outdir, inps.outfile) inps.outfile = os.path.abspath(inps.outfile) #delete geo_*.h5 files multitrack_utilities.delete_tmpgeo(inps.outdir, 'S1_', '.he5') multitrack_utilities.delete_tmpgeo(inps.outdir, 'geo_', '.h5') return
def calculate_delay_timeseries(tropo_file, dis_file, geom_file, GACOS_dir): """calculate delay time-series and write to HDF5 file""" ## get list of dates atr = readfile.read_attribute(dis_file) ftype = atr['FILE_TYPE'] if ftype == 'timeseries': date_list = timeseries(dis_file).get_date_list() elif ftype == '.unw': date12 = readfile.read_attribute(dis_file)['DATE12'] date_list = ptime.yyyymmdd(date12.split('-')) else: raise ValueError( 'un-supported displacement file type: {}'.format(ftype)) # list of dates --> list of ztd files ztd_files = [ os.path.join(GACOS_dir, '{}.ztd'.format(i)) for i in date_list ] # check missing ztd files flag = np.ones(len(date_list), dtype=np.bool_) for i in range(len(date_list)): if not os.path.isfile(ztd_files[i]): print('WARNING: {} file not found, ignore it and continue'.format( ztd_files[i])) flag[i] = False if np.any(flag == 0): date_list = np.array(date_list)[flag].tolist() ztd_files = np.array(ztd_files)[flag].tolist() ## update_mode def get_dataset_size(fname): atr = readfile.read_attribute(fname) return (atr['LENGTH'], atr['WIDTH']) def run_or_skip(ztd_files, tropo_file, geom_file): print('update mode: ON') print('output file: {}'.format(tropo_file)) flag = 'skip' # check existance and modification time if ut.run_or_skip(out_file=tropo_file, in_file=ztd_files, print_msg=False) == 'run': flag = 'run' print( '1) output file either do NOT exist or is NOT newer than all ZTD files.' ) else: print('1) output file exists and is newer than all ZTD files.') # check dataset size in space / time date_list = [str(re.findall('\d{8}', i)[0]) for i in ztd_files] if (get_dataset_size(tropo_file) != get_dataset_size(geom_file) or any(i not in timeseries(tropo_file).get_date_list() for i in date_list)): flag = 'run' print( '2) output file does NOT have the same len/wid as the geometry file {} or does NOT contain all dates' .format(geom_file)) else: print( '2) output file has the same len/wid as the geometry file and contains all dates' ) # check if output file is fully written with h5py.File(tropo_file, 'r') as f: if np.all(f['timeseries'][-1, :, :] == 0): flag = 'run' print('3) output file is NOT fully written.') else: print('3) output file is fully written.') # result print('run or skip: {}'.format(flag)) return flag if run_or_skip(ztd_files, tropo_file, geom_file) == 'skip': return ## prepare output file # metadata atr['FILE_TYPE'] = 'timeseries' atr['UNIT'] = 'm' # remove metadata related with double reference # because absolute delay is calculated and saved for key in ['REF_DATE', 'REF_X', 'REF_Y', 'REF_LAT', 'REF_LON']: if key in atr.keys(): atr.pop(key) # instantiate time-series length, width = int(atr['LENGTH']), int(atr['WIDTH']) num_date = len(date_list) dates = np.array(date_list, dtype=np.string_) ds_name_dict = { "date": [dates.dtype, (num_date, ), dates], "timeseries": [np.float32, (num_date, length, width), None], } writefile.layout_hdf5(tropo_file, ds_name_dict, metadata=atr) ## calculate phase delay # read geometry print('read incidenceAngle from file: {}'.format(geom_file)) inc_angle = readfile.read(geom_file, datasetName='incidenceAngle')[0] cos_inc_angle = np.cos(inc_angle * np.pi / 180.0) if 'Y_FIRST' in atr.keys(): pts_new = None else: # pixel coordinates in geometry file print('get pixel coordinates in geometry file') lats, lons = ut.get_lat_lon(atr, geom_file) pts_new = np.hstack((lats.reshape(-1, 1), lons.reshape(-1, 1))) # loop for date-by-date IO prog_bar = ptime.progressBar(maxValue=num_date) for i in range(num_date): date_str = date_list[i] ztd_file = ztd_files[i] # calc delay if 'Y_FIRST' in atr.keys(): delay = get_delay_geo(ztd_file, atr, cos_inc_angle) else: delay = get_delay_radar(ztd_file, cos_inc_angle, pts_new) # write delay to file block = [i, i + 1, 0, length, 0, width] writefile.write_hdf5_block(tropo_file, data=delay, datasetName='timeseries', block=block, print_msg=False) prog_bar.update(i + 1, suffix=os.path.basename(ztd_file)) prog_bar.close() return tropo_file
def calc_delay_timeseries(inps): """Calculate delay time-series and write it to HDF5 file. Parameters: inps : namespace, all input parameters Returns: tropo_file : str, file name of ECMWF.h5 """ def get_dataset_size(fname): atr = readfile.read_attribute(fname) shape = (int(atr['LENGTH']), int(atr['WIDTH'])) return shape def run_or_skip(grib_files, tropo_file, geom_file): print('update mode: ON') print('output file: {}'.format(tropo_file)) flag = 'skip' # check existance and modification time if ut.run_or_skip(out_file=tropo_file, in_file=grib_files, print_msg=False) == 'run': flag = 'run' print( '1) output file either do NOT exist or is NOT newer than all GRIB files.' ) else: print('1) output file exists and is newer than all GRIB files.') # check dataset size in space / time date_list = [ str(re.findall('\d{8}', os.path.basename(i))[0]) for i in grib_files ] if (get_dataset_size(tropo_file) != get_dataset_size(geom_file) or any(i not in timeseries(tropo_file).get_date_list() for i in date_list)): flag = 'run' print( '2) output file does NOT have the same len/wid as the geometry file {} or does NOT contain all dates' .format(geom_file)) else: print( '2) output file has the same len/wid as the geometry file and contains all dates' ) # check if output file is fully written with h5py.File(tropo_file, 'r') as f: if np.all(f['timeseries'][-1, :, :] == 0): flag = 'run' print('3) output file is NOT fully written.') else: print('3) output file is fully written.') # result print('run or skip: {}'.format(flag)) return flag if run_or_skip(inps.grib_files, inps.tropo_file, inps.geom_file) == 'skip': return ## 1. prepare geometry data geom_obj = geometry(inps.geom_file) geom_obj.open() inps.inc = geom_obj.read(datasetName='incidenceAngle') inps.dem = geom_obj.read(datasetName='height') # for testing if inps.custom_height: print( 'use input custom height of {} m for vertical integration'.format( inps.custom_height)) inps.dem[:] = inps.custom_height if 'latitude' in geom_obj.datasetNames: # for lookup table in radar-coord (isce, doris) inps.lat = geom_obj.read(datasetName='latitude') inps.lon = geom_obj.read(datasetName='longitude') elif 'Y_FIRST' in geom_obj.metadata: # for lookup table in geo-coded (gamma, roipac) and obs. in geo-coord inps.lat, inps.lon = ut.get_lat_lon(geom_obj.metadata) # convert coordinates to lat/lon, e.g. from UTM for ASF HyPP3 if not geom_obj.metadata['Y_UNIT'].startswith('deg'): inps.lat, inps.lon = ut.to_latlon(inps.atr['OG_FILE_PATH'], inps.lon, inps.lat) else: # for lookup table in geo-coded (gamma, roipac) and obs. in radar-coord inps.lat, inps.lon = ut.get_lat_lon_rdc(inps.atr) # mask of valid pixels mask = np.multiply(inps.inc != 0, ~np.isnan(inps.inc)) ## 2. prepare output file # metadata atr = inps.atr.copy() atr['FILE_TYPE'] = 'timeseries' atr['UNIT'] = 'm' # remove metadata related with double reference # because absolute delay is calculated and saved for key in ['REF_DATE', 'REF_X', 'REF_Y', 'REF_LAT', 'REF_LON']: if key in atr.keys(): atr.pop(key) # instantiate time-series length, width = int(atr['LENGTH']), int(atr['WIDTH']) num_date = len(inps.grib_files) date_list = [ str(re.findall('\d{8}', os.path.basename(i))[0]) for i in inps.grib_files ] dates = np.array(date_list, dtype=np.string_) ds_name_dict = { "date": [dates.dtype, (num_date, ), dates], "timeseries": [np.float32, (num_date, length, width), None], } writefile.layout_hdf5(inps.tropo_file, ds_name_dict, metadata=atr) ## 3. calculate phase delay print( '\n------------------------------------------------------------------------------' ) print( 'calculating absolute delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...' ) print('number of grib files used: {}'.format(num_date)) prog_bar = ptime.progressBar(maxValue=num_date, print_msg=~inps.verbose) for i in range(num_date): grib_file = inps.grib_files[i] # calc tropo delay tropo_data = get_delay(grib_file, tropo_model=inps.tropo_model, delay_type=inps.delay_type, dem=inps.dem, inc=inps.inc, lat=inps.lat, lon=inps.lon, mask=mask, verbose=inps.verbose) # write tropo delay to file block = [i, i + 1, 0, length, 0, width] writefile.write_hdf5_block(inps.tropo_file, data=tropo_data, datasetName='timeseries', block=block, print_msg=False) prog_bar.update(i + 1, suffix=os.path.basename(grib_file)) prog_bar.close() return inps.tropo_file
def main(iargs=None): inps = cmd_line_parse(iargs) inps.work_dir = os.path.abspath(os.path.dirname(inps.ts_file)) inps.cbar_file = os.path.join(inps.work_dir, 'google_earth_cbar.png') inps.star_file = os.path.join(inps.work_dir, "star.png") inps.dot_file = os.path.join(inps.work_dir, "shaded_dot.png") inps.dygraph_file = os.path.join(inps.work_dir, "dygraph-combined.js") inps.kml_data_dir = os.path.join(inps.work_dir, 'kml_data') ## Define file names inps.outfile_base = plot.auto_figure_title(inps.ts_file, inps_dict=vars(inps)) kml_master_file = os.path.join(inps.work_dir, '{}_master.kml'.format(inps.outfile_base)) kmz_file = os.path.join(inps.work_dir, '{}.kmz'.format(inps.outfile_base)) ## read data ts_obj = timeseries(inps.ts_file) ts_obj.open() length, width = ts_obj.length, ts_obj.width inps.metadata = ts_obj.metadata lats, lons = ut.get_lat_lon(ts_obj.metadata) print('input data shape in row/col: {}/{}'.format(length, width)) vel = readfile.read(inps.vel_file, datasetName='velocity')[0] * 100. # Set min/max velocity for colormap if inps.vlim is None: inps.vlim = [np.nanmin(vel), np.nanmax(vel)] ##--------- Create master KML file with network links to data KML files --------------## kml_master_doc = KML.Document() # 1 Create Overlay element for colorbar cbar_overlay = generate_cbar_element(cbar_file=inps.cbar_file, vmin=inps.vlim[0], vmax=inps.vlim[1], cmap=inps.colormap) kml_master_doc.append(cbar_overlay) # 2 Generate the placemark for the Reference Pixel ref_point = create_reference_point_element(inps, lats, lons, ts_obj) print('add reference point.') ref_folder = KML.Folder(KML.name("ReferencePoint")) ref_folder.append(ref_point) kml_master_doc.append(ref_folder) # 3 Create data folder to contain actual data elements net_link1 = generate_network_link(inps, ts_obj, step=inps.steps[0], lod=(0, inps.lods[0])) net_link2 = generate_network_link(inps, ts_obj, step=inps.steps[1], lod=(inps.lods[0], inps.lods[1])) net_link3 = generate_network_link(inps, ts_obj, step=inps.steps[2], lod=(inps.lods[1], inps.lods[2])) # 3.3 Append network links to data folder data_folder = KML.Folder(KML.name("Data")) data_folder.append(net_link1) data_folder.append(net_link2) data_folder.append(net_link3) kml_master_doc.append(data_folder) ##---------------------------- Write master KML file ------------------------------## print('-'*30) print('writing ' + kml_master_file) kml_master = KML.kml() kml_master.append(kml_master_doc) with open(kml_master_file, 'w') as f: f.write(etree.tostring(kml_master, pretty_print=True).decode('utf-8')) ## Copy auxiliary files res_dir = os.path.join(os.path.dirname(__file__), "../docs/resources") for fname in [inps.star_file, inps.dot_file, inps.dygraph_file]: src_file = os.path.join(res_dir, os.path.basename(fname)) shutil.copy2(src_file, inps.work_dir) print("copy {} to the local directory".format(src_file)) ## Generate KMZ file kml_files_str = '' for fname in [kml_master_file, inps.kml_data_dir, inps.cbar_file,inps.dygraph_file, inps.dot_file, inps.star_file]: kml_files_str += ' {}'.format(os.path.basename(fname)) cmd = 'cd {}; zip -r {} {}'.format(inps.work_dir, kmz_file, kml_files_str) print('writing {} from kml files'.format(kmz_file)) os.system(cmd) ## Remove extra files from file tree after KMZ generation cmd = 'cd {}; rm -r {}'.format(inps.work_dir, kml_files_str) os.system(cmd) print('Done.') print('Open {} in Google Earth!'.format(kmz_file)) return
def read_data(inps): """ Returns: defo: 2D np.array with in-valid/masked-out pixel in NaN """ # metadata inps.metadata = readfile.read_attribute(inps.file) k = inps.metadata['FILE_TYPE'] inps.range2phase = -4. * np.pi / float(inps.metadata['WAVELENGTH']) ext = os.path.splitext(inps.file)[1] # mask if inps.mask_file: inps.mask = readfile.read(inps.mask_file)[0] else: inps.mask = np.ones( (int(inps.metadata['LENGTH']), int(inps.metadata['WIDTH'])), dtype=np.bool_) # data if k in ['.unw', 'velocity']: inps.phase = readfile.read(inps.file)[0] if k == 'velocity': # velocity to displacement date1, date2 = inps.metadata['DATE12'].split('_') dt1, dt2 = ptime.date_list2vector([date1, date2])[0] inps.phase *= (dt2 - dt1).days / 365.25 # displacement to phase inps.phase *= inps.range2phase # update mask to exclude pixel with NaN value inps.mask *= ~np.isnan(inps.phase) # set all masked out pixel to NaN inps.phase[inps.mask == 0] = np.nan else: raise ValueError("input file not support yet: {}".format(k)) print('number of pixels: {}'.format(np.sum(inps.mask))) # change reference point if inps.ref_lalo: coord = ut.coordinate(inps.metadata) ref_lat, ref_lon = inps.ref_lalo ref_y, ref_x = coord.geo2radar(ref_lat, ref_lon)[0:2] # update data inps.phase -= inps.phase[ref_y, ref_x] # update metadata inps.metadata['REF_LAT'] = ref_lat inps.metadata['REF_LON'] = ref_lon inps.metadata['REF_Y'] = ref_y inps.metadata['REF_X'] = ref_x # read geometry inps.lat, inps.lon = ut.get_lat_lon(inps.metadata) inps.inc_angle = readfile.read(inps.geom_file, datasetName='incidenceAngle')[0] inps.head_angle = np.ones(inps.inc_angle.shape, dtype=np.float32) * float( inps.metadata['HEADING']) inps.lat[inps.mask == 0] = np.nan inps.lon[inps.mask == 0] = np.nan inps.inc_angle[inps.mask == 0] = np.nan inps.head_angle[inps.mask == 0] = np.nan # output filename if not inps.outfile: out_dir = os.path.dirname(inps.file) proj_name = sensor.project_name2sensor_name(out_dir)[1] if not proj_name: raise ValueError('No custom/auto output filename found.') inps.outfile = '{}_{}.mat'.format(proj_name, inps.metadata['DATE12']) inps.outfile = os.path.join(out_dir, inps.outfile) inps.outfile = os.path.abspath(inps.outfile) return