def make_bbox(inp_bbox):
"""Make a WKT from SNWE or a shapefile"""
if inp_bbox is None:
return None
from shapely.geometry import Polygon
if op.exists(op.abspath(inps.bbox)):
from ARIAtools.shapefile_util import open_shapefile
ring = open_shapefile(inps.bbox, 0, 0).exterior
poly = Polygon(ring)
else:
try:
S, N, W, E = [float(i) for i in inps.bbox.split()]
## adjust for degrees easting / northing (0 - 360 / 0:180)
if W > 180:
W -= 360
print('AdjustedW')
if E > 180:
E -= 360
print('AdjustedE')
if N > 90:
N -= 90
S -= 90
print('Adjusted N/S')
except:
raise Exception(
'Cannot understand the --bbox argument. '
'Input string was entered incorrectly or path does not '
'exist.')
poly = Polygon([(W, N), (W, S), (E, S), (E, N)])
return poly
def __init__(self,
product_dict,
workdir='./',
bbox_file=None,
prods_TOTbbox=None,
mask=None,
outputFormat='ENVI',
croptounion=False,
num_threads='2'):
# Pass inputs, and initialize list of pairs
self.product_dict = product_dict
self.bbox_file = bbox_file
self.workdir = os.path.join(workdir, 'figures')
self.prods_TOTbbox = prods_TOTbbox
self.mask = mask
self.outputFormat = outputFormat
self.croptounion = croptounion
self.num_threads = num_threads
self.pairs = None
self.mask_ext = '_mask' if self.mask is not None else ''
if self.bbox_file:
self.bbox_file = open_shapefile(bbox_file, 0, 0).bounds
if self.outputFormat.upper() == 'VRT':
self.outputFormat = 'ENVI'
os.makedirs(self.workdir, exist_ok=True)
def __init__(self,
product_dict,
workdir='./',
bbox_file=None,
prods_TOTbbox=None,
mask=None,
outputFormat='ENVI',
croptounion=False):
# Pass inputs, and initialize list of pairs
self.product_dict = product_dict
self.bbox_file = bbox_file
if self.bbox_file:
self.bbox_file = open_shapefile(bbox_file, 0, 0).bounds
self.workdir = os.path.join(workdir, 'figures')
self.prods_TOTbbox = prods_TOTbbox
self.mask = mask
self.outputFormat = outputFormat
self.croptounion = croptounion
# File must be physically extracted, cannot proceed with VRT format. Defaulting to ENVI format.
if self.outputFormat == 'VRT':
self.outputFormat = 'ENVI'
# create workdir if it doesn't exist
if not os.path.exists(self.workdir):
os.mkdir(self.workdir)
self.pairs = None
self.mask_ext = '_mask' if self.mask is not None else ''
def __readproduct__(self, file):
'''
Read product, determine expected layer names based off of version number, and populate corresponding product dictionary accordingly.
'''
### Get standard product version from file
# If netcdf with groups
try:
version = str(gdal.Open(file).GetMetadataItem('NC_GLOBAL#version'))
except:
print('{} is not a supported file type... skipping'.format(file))
return []
# If netcdf with nogroups
if version == str(None):
version = str(gdal.Open(file).GetMetadataItem('version'))
### Get lists of radarmetadata/layer keys for this file version
rmdkeys, sdskeys = self.__mappingVersion__(file, version)
if self.bbox is not None:
# Open standard product bbox
file_bbox = open_shapefile(
'NETCDF:"' + file + '":' + sdskeys[0], 'productBoundingBox', 1
) ##SS => We should track the projection of the shapefile. i.e. in case this changes in the product
# Only generate dictionaries if there is spatial overlap with user bbox
if file_bbox.intersects(self.bbox):
product_dicts = [self.__mappingData__(file, rmdkeys, sdskeys)]
else:
product_dicts = []
pass
# If no bbox specified, just pass dictionaries
else:
product_dicts = [self.__mappingData__(file, rmdkeys, sdskeys)]
return product_dicts
def __init__(self, filearg, bbox=None, workdir='./', verbose=False):
# If user wants verbose mode
self.verbose=verbose
# Parse through file(s)/bbox input
self.files = []
self.products = []
# Track bbox file
self.bbox_file = None
# Pair name for layer extraction
self.pairname = None
### Determine if file input is single file, a list, or wildcard.
# If list of files
if len([str(val) for val in filearg.split(',')])>1:
self.files=[str(val) for val in filearg.split(',')]
# If single file or wildcard
else:
# If single file
if os.path.isfile(filearg):
self.files=[filearg]
# If wildcard
else:
self.files=self.glob.glob(os.path.expanduser(os.path.expandvars(filearg)))
# Convert relative paths to absolute paths
self.files=[os.path.normpath(os.path.join(os.getcwd(),i)) if not os.path.isabs(i) else i for i in self.files]
if len(self.files)==0:
raise Exception('No file match found')
# If specified workdir doesn't exist, create it
if not os.path.exists(workdir):
os.mkdir(workdir)
### Check if bbox input is valid list or shapefile.
if bbox is not None:
# If list
if isinstance ([str(val) for val in bbox.split()], list) and not os.path.isfile(bbox):
from shapely.geometry import Polygon
try:
bbox = [float(val) for val in bbox.split()]
except:
raise Exception('Cannot understand the --bbox argument. String input is incorrect or path does not exist.')
# Use shapely to make list
self.bbox = Polygon(np.column_stack((np.array([bbox[2],bbox[3],bbox[3],bbox[2],bbox[2]]),
np.array([bbox[0],bbox[0],bbox[1],bbox[1],bbox[0]])))) #Pass lons/lats to create polygon
# Save polygon in shapefile
save_shapefile(os.path.join(workdir,'user_bbox.shp'), self.bbox, 'GeoJSON')
self.bbox_file=os.path.join(workdir,'user_bbox.shp')
print("Shapefile %s created for input user bounds."%os.path.join(workdir,'user_bbox.shp'))
# If shapefile
elif os.path.isfile(bbox):
self.bbox = open_shapefile(bbox, 0, 0) ##SS => We should track the projection of the shapefile. i.e. if user provides this in e.g. UTM etc.
self.bbox_file = bbox
else:
raise Exception('bbox input neither valid list nor file')
else:
self.bbox=None
### Report dictionaries for all valid products
self.__run__()
def _get_bbox(self):
if op.exists(op.abspath(self.inps.bbox)):
from ARIAtools.shapefile_util import open_shapefile
bounds = open_shapefile(self.inps.bbox, 0, 0).bounds
W, S, E, N = [str(i) for i in bounds]
else:
try:
S, N, W, E = self.inps.bbox.split()
except:
raise Exception(
'Cannot understand the --bbox argument. Input string was entered incorrectly or path does not exist.'
)
return ','.join([W, S, E, N])
def __readproduct__(self, fname):
"""
Read products.
Read product, determine expected layer names based off of version
number, and populate corresponding product dictionary accordingly.
"""
### Get standard product version from file
try:
#version accessed differently between URL vs downloaded product
version = str(
gdal.Open(fname).GetMetadataItem('NC_GLOBAL#version'))
except:
log.warning('%s is not a supported file type... skipping', fname)
return []
### Get lists of radarmetadata/layer keys for this file version
fname = 'NETCDF:"' + fname
rmdkeys, sdskeys = self.__mappingVersion__(fname, version)
# Open standard product bbox
if self.bbox is not None:
file_bbox = open_shapefile(fname + '":' + sdskeys[0],
'productBoundingBox', 1)
# Only generate dictionaries if there is spatial overlap with user bbox
if file_bbox.intersects(self.bbox):
product_dicts = [self.__mappingData__( \
fname, rmdkeys, sdskeys, version)]
else:
product_dicts = []
# If no bbox specified, just pass dictionaries
else:
product_dicts = [self.__mappingData__( \
fname, rmdkeys, sdskeys, version)]
return product_dicts
def plot_extents(self, figwidth=6.4):
""" Make plot of track extents vs bbox/common track extent. """
# Figure size based on number of products
if figwidth in ['standard', 'narrow']:
# Use standard figure width
figwidth = 6.4
elif figwidth in ['wide', 'auto']:
# Automatically adjust figure width
figwidth = 0.17 * len(self.product_dict[0]) + 6.4
else:
# Else, use user-specified width of figure
figwidth = float(figwidth)
ax = plt.figure(figsize=(figwidth, 4.8)).add_subplot(111)
# Iterate through all IFGs
S_extent = []
N_extent = []
for i in enumerate(self.product_dict[0]):
prods_bbox = open_shapefile(i[1][0], 0, 0).bounds
S_extent.append(prods_bbox[1])
N_extent.append(prods_bbox[3])
# Plot IFG extent bounds in latitude
ax.plot([self.product_dict[1][i[0]][0]] * 2,
list(prods_bbox[1::2]),
'ko',
markersize=10)
# Plot IFG extent line connecting bounds in latitude
ax.plot([self.product_dict[1][i[0]][0]] * 2,
list(prods_bbox[1::2]),
color='0.5',
linestyle='--')
# Plot bounds of common track extent
if self.croptounion:
S_extent = min(S_extent)
N_extent = max(N_extent)
if [self.bbox_file[1], self.bbox_file[3]
] != [S_extent, N_extent] and S_extent != N_extent:
ax.axhline(y=S_extent,
color='r',
linestyle=':',
label="extent of union")
ax.axhline(y=N_extent, color='r', linestyle=':')
else:
S_extent = max(S_extent)
N_extent = min(N_extent)
if [self.bbox_file[1], self.bbox_file[3]
] != [S_extent, N_extent] and S_extent != N_extent:
ax.axhline(y=S_extent,
color='r',
linestyle=':',
label="extent of intersection")
ax.axhline(y=N_extent, color='r', linestyle=':')
# Plot bounds of final track extent all IFGs will be cropped to
ax.axhline(y=self.bbox_file[1],
color='b',
linestyle='--',
label="bounding box")
ax.axhline(y=self.bbox_file[3], color='b', linestyle='--')
# add legend
plt.legend(bbox_to_anchor=(1.05, 1),
loc='upper left',
borderaxespad=0.)
# defining the axis labels
ax.set_ylabel('Latitude', weight='bold')
ax.set_xlabel('Interferograms', weight='bold')
ax.set_title('Interferogram lat extents', weight='bold')
plt.xticks(rotation=90)
plt.tight_layout()
# saving the figure
plt.savefig(os.path.join(self.workdir, 'lat_extents.eps'))
plt.savefig(os.path.join(self.workdir, 'lat_extents.png'))
plt.close()
return
def export_products(full_product_dict,
bbox_file,
prods_TOTbbox,
layers,
dem=None,
lat=None,
lon=None,
mask=None,
outDir='./',
outputFormat='VRT',
stitchMethodType='overlap',
verbose=None):
"""
Export layer and 2D meta-data layers (at the product resolution).
The function finalize_metadata is called to derive the 2D metadata layer. Dem/lat/lon arrays must be passed for this process.
The keys specify which layer to extract from the dictionary.
All products are cropped by the bounds from the input bbox_file, and clipped to the track extent denoted by the input prods_TOTbbox.
Optionally, a user may pass a mask-file.
"""
if not layers: return # only bbox
bounds = open_shapefile(bbox_file, 0, 0).bounds
# Loop through user expected layers
for key in layers:
product_dict = [[j[key] for j in full_product_dict],
[j["pair_name"] for j in full_product_dict]]
workdir = os.path.join(outDir, key)
# If specified workdir doesn't exist, create it
if not os.path.exists(workdir):
os.mkdir(workdir)
# Iterate through all IFGs
print("Generating: " + key)
for i, j in enumerate(product_dict[0]):
outname = os.path.abspath(
os.path.join(workdir, product_dict[1][i][0]))
# Extract/crop metadata layers
if any(":/science/grids/imagingGeometry" in s for s in j):
gdal.BuildVRT(outname + '.vrt', j)
if dem is None:
raise Exception(
'No DEM input specified. Cannot extract 3D imaging geometry layers without DEM to intersect with.'
)
# Check if height layers are consistent, and if not exit with error
if len(
set([
gdal.Open(i).GetMetadataItem(
'NETCDF_DIM_heightsMeta_VALUES') for i in j
])) == 1:
gdal.Open(outname + '.vrt').SetMetadataItem(
'NETCDF_DIM_heightsMeta_VALUES',
gdal.Open(j[0]).GetMetadataItem(
'NETCDF_DIM_heightsMeta_VALUES'))
else:
raise Exception(
'Inconsistent heights for metadata layer(s) ', j,
' corresponding heights: ', [
gdal.Open(i).GetMetadataItem(
'NETCDF_DIM_heightsMeta_VALUES') for i in j
])
# Pass metadata layer VRT, with DEM filename and output name to interpolate/intersect with DEM before cropping
finalize_metadata(outname,
open_shapefile(bbox_file, 0, 0).bounds,
prods_TOTbbox,
dem,
lat,
lon,
mask,
outputFormat,
verbose=verbose)
# Extract/crop full res layers, except for "unw" and "conn_comp" which requires advanced stitching
elif key != 'unwrappedPhase' and key != 'connectedComponents':
if outputFormat == 'VRT' and mask is None:
# building the virtual vrt
gdal.BuildVRT(outname + "_uncropped" + '.vrt', j)
# building the cropped vrt
gdal.Warp(outname + '.vrt',
outname + "_uncropped" + '.vrt',
options=gdal.WarpOptions(
format=outputFormat,
cutlineDSName=prods_TOTbbox,
outputBounds=bounds))
else:
# building the VRT
gdal.BuildVRT(outname + '.vrt', j)
# Mask specified, so file must be physically extracted, cannot proceed with VRT format. Defaulting to ENVI format.
if outputFormat == 'VRT' and mask is not None:
outputFormat = 'ENVI'
gdal.Warp(outname,
outname + '.vrt',
options=gdal.WarpOptions(
format=outputFormat,
cutlineDSName=prods_TOTbbox,
outputBounds=bounds))
# Update VRT
gdal.Translate(outname + '.vrt',
outname,
options=gdal.TranslateOptions(format="VRT"))
# Apply mask (if specified).
if mask is not None:
update_file = gdal.Open(outname, gdal.GA_Update)
update_file = update_file.GetRasterBand(1).WriteArray(
mask * gdal.Open(outname + '.vrt').ReadAsArray())
update_file = None
# Extract/crop "unw" and "conn_comp" layers leveraging the two stage unwrapper
else:
# Check if unw phase and connected components are already generated
if not os.path.exists(
os.path.join(
outDir, 'unwrappedPhase',
product_dict[1][i][0])) or not os.path.exists(
os.path.join(outDir, 'connectedComponents',
product_dict[1][i][0])):
# extract the inputs required for stitching of unwrapped and connected component files
unw_files = full_product_dict[i]['unwrappedPhase']
conn_files = full_product_dict[i]['connectedComponents']
prod_bbox_files = full_product_dict[i][
'productBoundingBoxFrames']
# based on the key define the output directories
outFileUnw = os.path.join(outDir, 'unwrappedPhase',
product_dict[1][i][0])
outFileConnComp = os.path.join(outDir,
'connectedComponents',
product_dict[1][i][0])
# calling the stitching methods
if stitchMethodType == 'overlap':
product_stitch_overlap(unw_files,
conn_files,
prod_bbox_files,
bounds,
prods_TOTbbox,
outFileUnw=outFileUnw,
outFileConnComp=outFileConnComp,
mask=mask,
outputFormat=outputFormat,
verbose=verbose)
elif stitchMethodType == '2stage':
product_stitch_2stage(unw_files,
conn_files,
bounds,
prods_TOTbbox,
outFileUnw=outFileUnw,
outFileConnComp=outFileConnComp,
mask=mask,
outputFormat=outputFormat,
verbose=verbose)
return
def merged_productbbox(product_dict,
workdir='./',
bbox_file=None,
croptounion=False):
'''
Extract/merge productBoundingBox layers for each pair and update dict, report common track bbox (default is to take common intersection, but user may specify union), and expected shape for DEM.
'''
# Import functions
from ARIAtools.shapefile_util import save_shapefile
# If specified workdir doesn't exist, create it
if not os.path.exists(workdir):
os.mkdir(workdir)
# Extract/merge productBoundingBox layers
for scene in product_dict:
# Get pair name, expected in dictionary
pair_name = scene["pair_name"][0]
outname = os.path.join(workdir, pair_name + '.shp')
# Create union of productBoundingBox layers
for frame in scene["productBoundingBox"]:
prods_bbox = open_shapefile(frame, 'productBoundingBox', 1)
if os.path.exists(outname):
union_bbox = open_shapefile(outname, 0, 0)
prods_bbox = prods_bbox.union(union_bbox)
save_shapefile(
outname, prods_bbox, 'GeoJSON'
) ##SS can we track and provide the proj information of the geojson?
scene["productBoundingBox"] = [outname]
prods_TOTbbox = os.path.join(workdir, 'productBoundingBox.shp')
# Initiate intersection file with first product
# this is for different scenes
save_shapefile(
prods_TOTbbox,
open_shapefile(product_dict[0]['productBoundingBox'][0], 0, 0),
'GeoJSON')
for scene in product_dict[1:]:
prods_bbox = open_shapefile(scene['productBoundingBox'][0], 0, 0)
total_bbox = open_shapefile(prods_TOTbbox, 0, 0)
# Generate footprint for the union of all products
if croptounion:
prods_bbox = prods_bbox.union(total_bbox)
# Generate footprint for the common intersection of all products
else:
prods_bbox = prods_bbox.intersection(total_bbox)
# Check if there is any common overlap
if prods_bbox.bounds == ():
raise Exception(
'No common overlap, footprint cannot be generated. Last scene checked: %s'
% (scene['productBoundingBox'][0]))
save_shapefile(prods_TOTbbox, prods_bbox, 'GeoJSON')
# If bbox specified, intersect with common track intersection/union
if bbox_file is not None:
user_bbox = open_shapefile(bbox_file, 0, 0)
total_bbox = open_shapefile(prods_TOTbbox, 0, 0)
user_bbox = user_bbox.intersection(total_bbox)
save_shapefile(prods_TOTbbox, user_bbox, 'GeoJSON')
# Estimate percentage of overlap with bbox
prods_bbox_area = open_shapefile(prods_TOTbbox, 0, 0).bounds
prods_bbox_area = (max(prods_bbox_area[0], prods_bbox_area[2]) -
min(prods_bbox_area[0], prods_bbox_area[2])) * (
max(prods_bbox_area[1], prods_bbox_area[3]) -
min(prods_bbox_area[1], prods_bbox_area[3]))
bbox_area = open_shapefile(bbox_file, 0, 0).bounds
bbox_area = (max(bbox_area[0], bbox_area[2]) - min(
bbox_area[0], bbox_area[2])) * (max(bbox_area[1], bbox_area[3]) -
min(bbox_area[1], bbox_area[3]))
per_overlap = (prods_bbox_area / bbox_area) * 100
if per_overlap < 50.:
print(
"WARNING: Common track extent only has %d%% overlap with bbox"
% per_overlap + '\n')
else:
bbox_file = prods_TOTbbox
# Warp the first scene with the output-bounds defined above
ds = gdal.Warp('',
gdal.BuildVRT('', product_dict[0]['unwrappedPhase'][0]),
options=gdal.WarpOptions(format="MEM",
outputBounds=open_shapefile(
bbox_file, 0, 0).bounds))
# Get shape of full res layers
arrshape = [ds.RasterYSize, ds.RasterXSize]
# Get projection of full res layers
proj = ds.GetProjection()
ds = None
return product_dict, bbox_file, prods_TOTbbox, arrshape, proj
def prep_mask(product_dict,
maskfilename,
bbox_file,
prods_TOTbbox,
proj,
amp_thresh=None,
arrshape=None,
workdir='./',
outputFormat='ENVI'):
'''
Function to load and export mask file.
If "Download" flag is specified, GSHHS water mask will be donwloaded on the fly.
'''
# Import functions
from ARIAtools.vrtmanager import renderVRT
import glob
_world_watermask = [
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L1.shp',
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L2.shp',
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L3.shp',
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L4.shp',
' /vsizip/vsicurl/https://osmdata.openstreetmap.de/download/land-polygons-complete-4326.zip/land-polygons-complete-4326/land_polygons.shp'
]
# Get bounds of user bbox_file
bounds = open_shapefile(bbox_file, 0, 0).bounds
# File must be physically extracted, cannot proceed with VRT format. Defaulting to ENVI format.
if outputFormat == 'VRT':
outputFormat = 'ENVI'
# Download mask
if maskfilename.lower() == 'download':
maskfilename = os.path.join(workdir, 'watermask' + '.msk')
###Make coastlines/islands union VRT
os.system('ogrmerge.py -o ' +
os.path.join(workdir, 'watermsk_shorelines.vrt') +
''.join(_world_watermask[::2]) +
' -field_strategy Union -f VRT -single')
###Make lakes/ponds union VRT
os.system('ogrmerge.py -o ' +
os.path.join(workdir, 'watermsk_lakes.vrt') +
''.join(_world_watermask[1::2]) +
' -field_strategy Union -f VRT -single')
###Initiate water-mask with coastlines/islands union VRT
gdal.Rasterize(maskfilename,
os.path.join(workdir, 'watermsk_shorelines.vrt'),
options=gdal.RasterizeOptions(format=outputFormat,
outputBounds=bounds,
outputType=gdal.GDT_Byte,
width=arrshape[1],
height=arrshape[0],
burnValues=[1],
layers='merged'))
gdal.Open(maskfilename, gdal.GA_Update).SetProjection(proj)
gdal.Translate(maskfilename + '.vrt',
maskfilename,
options=gdal.TranslateOptions(format="VRT"))
###Must take inverse of lakes/ponds union because of opposite designation (1 for water, 0 for land) as desired (0 for water, 1 for land)
lake_masks = gdal.Rasterize(
'',
os.path.join(workdir, 'watermsk_lakes.vrt'),
options=gdal.RasterizeOptions(format='MEM',
outputBounds=bounds,
outputType=gdal.GDT_Byte,
width=arrshape[1],
height=arrshape[0],
burnValues=[1],
layers='merged',
inverse=True))
lake_masks.SetProjection(proj)
lake_masks = lake_masks.ReadAsArray()
if amp_thresh:
###Make average amplitude mask
# Iterate through all IFGs
for i, j in enumerate(product_dict[0]):
amp_file = gdal.Warp('',
j,
options=gdal.WarpOptions(
format="MEM",
cutlineDSName=prods_TOTbbox,
outputBounds=bounds))
amp_file_arr = np.ma.masked_where(
amp_file.ReadAsArray() == amp_file.GetRasterBand(
1).GetNoDataValue(), amp_file.ReadAsArray())
# Iteratively update average amplitude file
# If looping through first amplitude file, nothing to sum so just save to file
if os.path.exists(os.path.join(workdir, 'avgamplitude.msk')):
amp_file = gdal.Open(
os.path.join(workdir, 'avgamplitude.msk'),
gdal.GA_Update)
amp_file = amp_file.GetRasterBand(1).WriteArray(
amp_file_arr + amp_file.ReadAsArray())
else:
renderVRT(
os.path.join(workdir, 'avgamplitude.msk'),
amp_file_arr,
geotrans=amp_file.GetGeoTransform(),
drivername=outputFormat,
gdal_fmt=amp_file_arr.dtype.name,
proj=amp_file.GetProjection(),
nodata=amp_file.GetRasterBand(1).GetNoDataValue())
amp_file = coh_val = amp_file_arr = None
# Take average of amplitude sum
amp_file = gdal.Open(os.path.join(workdir, 'avgamplitude.msk'),
gdal.GA_Update)
arr_mean = amp_file.ReadAsArray() / len(product_dict[0])
arr_mean = np.where(arr_mean < float(amp_thresh), 0, 1)
amp_file = amp_file.GetRasterBand(1).WriteArray(arr_mean)
amp_file = None
arr_mean = None
amp_file = gdal.Open(os.path.join(
workdir, 'avgamplitude.msk')).ReadAsArray()
else:
amp_file = np.ones((lake_masks.shape[0], lake_masks.shape[1]))
###Update water-mask with lakes/ponds union and average amplitude
update_file = gdal.Open(maskfilename, gdal.GA_Update)
update_file = update_file.GetRasterBand(1).WriteArray(
update_file.ReadAsArray() * lake_masks * amp_file)
print('Downloaded water-mask here: ' + maskfilename)
update_file = None
lake_masks = None
amp_file = None
#Delete temp files
os.remove(os.path.join(workdir, 'watermsk_shorelines.vrt'))
os.remove(os.path.join(workdir, 'watermsk_lakes.vrt'))
for i in glob.glob(os.path.join(workdir, 'avgamplitude.msk*')):
os.remove(i)
# Load mask
try:
mask = gdal.Warp('',
maskfilename,
options=gdal.WarpOptions(format="MEM",
cutlineDSName=prods_TOTbbox,
outputBounds=bounds,
dstNodata=0))
mask.SetProjection(proj)
# If no data value
if mask.GetRasterBand(1).GetNoDataValue():
mask = np.ma.masked_where(
mask.ReadAsArray() == mask.GetRasterBand(1).GetNoDataValue(),
mask.ReadAsArray())
else:
mask = mask.ReadAsArray()
except:
raise Exception('Failed to open user mask')
return mask
def prep_dem(demfilename,
bbox_file,
prods_TOTbbox,
proj,
arrshape=None,
workdir='./',
outputFormat='ENVI'):
'''
Function to load and export DEM, lat, lon arrays.
If "Download" flag is specified, DEM will be donwloaded on the fly.
'''
_world_dem = '/vsicurl/https://cloud.sdsc.edu/v1/AUTH_opentopography/Raster/SRTM_GL1_Ellip/SRTM_GL1_Ellip_srtm.vrt'
# Get bounds of user bbox_file
bounds = open_shapefile(bbox_file, 0, 0).bounds
# File must be physically extracted, cannot proceed with VRT format. Defaulting to ENVI format.
if outputFormat == 'VRT':
outputFormat = 'ENVI'
# Download DEM
if demfilename.lower() == 'download':
demfilename = os.path.join(workdir, 'SRTM_3arcsec' + '.dem')
gdal.Warp(demfilename,
_world_dem,
options=gdal.WarpOptions(format=outputFormat,
outputBounds=bounds,
outputType=gdal.GDT_Int16,
width=arrshape[1],
height=arrshape[0],
dstNodata=-32768.0,
srcNodata=-32768.0))
gdal.Open(demfilename, gdal.GA_Update).SetProjection(proj)
gdal.Translate(demfilename + '.vrt',
demfilename,
options=gdal.TranslateOptions(format="VRT")) #Make VRT
print('Downloaded 3 arc-sec SRTM DEM here: ' + demfilename)
# Load DEM and setup lat and lon arrays
try:
demfile = gdal.Warp('',
demfilename,
options=gdal.WarpOptions(
format="MEM",
cutlineDSName=prods_TOTbbox,
outputBounds=bounds,
dstNodata=0))
demfile.SetProjection(proj)
##SS Do we need lon lat if we would be doing gdal reproject using projection and transformation? See our earlier discussions.
# Define lat/lon arrays for fullres layers
Latitude = np.linspace(
demfile.GetGeoTransform()[3],
demfile.GetGeoTransform()[3] +
(demfile.GetGeoTransform()[5] * demfile.RasterYSize),
demfile.RasterYSize)
Latitude = np.repeat(Latitude[:, np.newaxis],
demfile.RasterXSize,
axis=1)
Longitude = np.linspace(
demfile.GetGeoTransform()[0],
demfile.GetGeoTransform()[0] +
(demfile.GetGeoTransform()[1] * demfile.RasterXSize),
demfile.RasterXSize)
Longitude = np.repeat(Longitude[:, np.newaxis],
demfile.RasterYSize,
axis=1)
Longitude = Longitude.transpose()
except:
raise Exception('Failed to open user DEM')
return demfilename, demfile, Latitude, Longitude
def __init__(self, filearg, bbox=None, workdir='./', verbose=False):
# If user wants verbose mode
# Parse through file(s)/bbox input
if verbose: logger.setLevel(logging.DEBUG)
self.files = []
self.products = []
# Track bbox file
self.bbox_file = None
# Pair name for layer extraction
self.pairname = None
### Determine if file input is single file, a list, or wildcard.
# If list of files
if len([str(val) for val in filearg.split(',')]) > 1:
self.files = [str(i) for i in filearg.split(',')]
# If wildcard
self.files = [
os.path.abspath(item) for sublist in [
self.glob.glob(os.path.expanduser(os.path.expandvars(i))
) if '*' in i else [i] for i in self.files
] for item in sublist
]
# If list of URLs provided
elif os.path.basename(filearg).endswith('.txt'):
with open(filearg, 'r') as fh:
self.files = [f.rstrip('\n') for f in fh.readlines()]
# If single file or wildcard
else:
# If single file
if os.path.isfile(filearg):
self.files = [filearg]
# If wildcard
else:
self.files = self.glob.glob(
os.path.expanduser(os.path.expandvars(filearg)))
# Convert relative paths to absolute paths
self.files = [os.path.abspath(i) for i in self.files]
# If URLs, append with '/vsicurl/'
self.files = [
'/vsicurl/{}'.format(i) if 'https://' in i else i
for i in self.files
]
#check if virtual file reader is being captured as netcdf
if any("https://" in i for i in self.files):
# must configure gdal to load URLs
gdal.SetConfigOption('GDAL_HTTP_COOKIEFILE', 'cookies.txt')
gdal.SetConfigOption('GDAL_HTTP_COOKIEJAR', 'cookies.txt')
#gdal.SetConfigOption('CPL_VSIL_CURL_CHUNK_SIZE','10485760')
gdal.SetConfigOption('VSI_CACHE', 'YES')
fmt = gdal.Open([s for s in self.files if 'https://' in s
][0]).GetDriver().GetDescription()
if fmt != 'netCDF':
raise Exception(
'System update required to read requested virtual products: Linux kernel >=4.3 and libnetcdf >=4.5'
)
#check if local file reader is being captured as netcdf
if any("https://" not in i for i in self.files):
fmt = gdal.Open([s for s in self.files if 'https://' not in s
][0]).GetDriver().GetDescription()
if fmt != 'netCDF':
raise Exception(
'System update required to read requested local products: Linux kernel >=4.3 and libnetcdf >=4.5'
)
if len(self.files) == 0:
raise Exception('No file match found')
# If specified workdir doesn't exist, create it
if not os.path.exists(workdir):
os.mkdir(workdir)
### Check if bbox input is valid list or shapefile.
if bbox is not None:
# If list
if isinstance([str(val) for val in bbox.split()],
list) and not os.path.isfile(bbox):
from shapely.geometry import Polygon
try:
bbox = [float(val) for val in bbox.split()]
except:
raise Exception(
'Cannot understand the --bbox argument. String input is incorrect or path does not exist.'
)
# Use shapely to make list
self.bbox = Polygon(
np.column_stack((np.array([
bbox[2], bbox[3], bbox[3], bbox[2], bbox[2]
]), np.array([bbox[0], bbox[0], bbox[1], bbox[1], bbox[0]
])))) #Pass lons/lats to create polygon
# Save polygon in shapefile
save_shapefile(os.path.join(workdir, 'user_bbox.json'),
self.bbox, 'GeoJSON')
self.bbox_file = os.path.join(workdir, 'user_bbox.json')
log.info("Shapefile %s created for input user bounds.",
os.path.join(workdir, 'user_bbox.json'))
# If shapefile
elif os.path.isfile(bbox):
self.bbox = open_shapefile(
bbox, 0, 0
) ##SS => We should track the projection of the shapefile. i.e. if user provides this in e.g. UTM etc.
self.bbox_file = bbox
else:
raise Exception('bbox input neither valid list nor file')
else:
self.bbox = None
### Report dictionaries for all valid products
self.__run__()
def plot_extents(self):
'''
Make plot of track extents vs bounding bbox/common track extent.
'''
ax = self.plt.figure().add_subplot(111)
#Iterate through all IFGs
S_extent = []
N_extent = []
for i, j in enumerate(self.product_dict[0]):
prods_bbox = open_shapefile(j[0], 0, 0).bounds
S_extent.append(prods_bbox[1])
N_extent.append(prods_bbox[3])
# Plot IFG extent bounds in latitude
ax.plot([self.product_dict[1][i][0]] * 2,
list(prods_bbox[1::2]),
'ko',
markersize=10)
# Plot IFG extent line connecting bounds in latitude
ax.plot([self.product_dict[1][i][0]] * 2,
list(prods_bbox[1::2]),
color='0.5',
linestyle='--')
# Plot bounds of common track extent
if self.croptounion:
S_extent = min(S_extent)
N_extent = max(N_extent)
if [self.bbox_file[1], self.bbox_file[3]
] != [S_extent, N_extent] and S_extent != N_extent:
ax.axhline(y=S_extent,
color='r',
linestyle=':',
label="extent of union")
ax.axhline(y=N_extent, color='r', linestyle=':')
else:
S_extent = max(S_extent)
N_extent = min(N_extent)
if [self.bbox_file[1], self.bbox_file[3]
] != [S_extent, N_extent] and S_extent != N_extent:
ax.axhline(y=S_extent,
color='r',
linestyle=':',
label="extent of intersection")
ax.axhline(y=N_extent, color='r', linestyle=':')
# Plot bounds of final track extent all IFGs will be cropped to
ax.axhline(y=self.bbox_file[1],
color='b',
linestyle='--',
label="bounding box")
ax.axhline(y=self.bbox_file[3], color='b', linestyle='--')
# add legend
self.plt.legend(bbox_to_anchor=(1.05, 1),
loc='upper left',
borderaxespad=0.)
# defining the axis labels
ax.set_ylabel('Latitude', weight='bold')
ax.set_xlabel('Interferograms', weight='bold')
ax.set_title('Interferogram lat extents', weight='bold')
self.plt.xticks(rotation=90)
self.plt.tight_layout()
# saving the figure
self.plt.savefig(os.path.join(self.workdir, 'lat_extents.eps'))
self.plt.close()
return
def prep_mask(product_dict,
maskfilename,
bbox_file,
prods_TOTbbox,
proj,
amp_thresh=None,
arrshape=None,
workdir='./',
outputFormat='ENVI',
num_threads='2'):
'''
Function to load and export mask file.
If "Download" flag is specified, GSHHS water mask will be donwloaded on the fly.
If the full resolution NLCD landcover data is given (NLCD...img) it will be cropped to match product
'''
# Import functions
from ARIAtools.vrtmanager import renderOGRVRT
_world_watermask = [
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L1.shp',
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L2.shp',
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L3.shp',
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L4.shp',
' /vsizip/vsicurl/https://osmdata.openstreetmap.de/download/land-polygons-complete-4326.zip/land-polygons-complete-4326/land_polygons.shp'
]
# If specified DEM subdirectory exists, delete contents
workdir = os.path.join(workdir, 'mask')
if os.path.exists(
workdir) and os.path.abspath(maskfilename) != os.path.abspath(
os.path.join(
workdir,
os.path.basename(maskfilename).split('.')
[0].split('uncropped')[0] + '.msk')) and os.path.abspath(
maskfilename) != os.path.abspath(
os.path.join(
workdir,
os.path.basename(maskfilename).split('.')[0] +
'.msk')) or maskfilename.lower() == 'download':
for i in glob.glob(os.path.join(workdir, '*.*')):
os.remove(i)
if not os.path.exists(workdir):
os.mkdir(workdir)
# Get bounds of user bbox_file
bounds = open_shapefile(bbox_file, 0, 0).bounds
# File must be physically extracted, cannot proceed with VRT format. Defaulting to ENVI format.
if outputFormat == 'VRT':
outputFormat = 'ENVI'
# Download mask
if maskfilename.lower() == 'download':
log.info("***Downloading water mask... ***")
maskfilename = os.path.join(workdir, 'watermask' + '.msk')
os.environ['CPL_ZIP_ENCODING'] = 'UTF-8'
###Make coastlines/islands union VRT
renderOGRVRT(os.path.join(workdir, 'watermsk_shorelines.vrt'),
_world_watermask[::2])
###Make lakes/ponds union VRT
renderOGRVRT(os.path.join(workdir, 'watermsk_lakes.vrt'),
_world_watermask[1::2])
###Initiate water-mask with coastlines/islands union VRT
# save uncropped mask
gdal.Rasterize(os.path.join(workdir, 'watermask_uncropped.msk'),
os.path.join(workdir, 'watermsk_shorelines.vrt'),
options=gdal.RasterizeOptions(format=outputFormat,
outputBounds=bounds,
outputType=gdal.GDT_Byte,
width=arrshape[1],
height=arrshape[0],
burnValues=[1],
layers='merged'))
gdal.Translate(os.path.join(workdir, 'watermask_uncropped.msk.vrt'),
os.path.join(workdir, 'watermask_uncropped.msk'),
options=gdal.TranslateOptions(format="VRT"))
# save cropped mask
gdal.Warp(maskfilename,
os.path.join(workdir, 'watermask_uncropped.msk.vrt'),
options=gdal.WarpOptions(
format=outputFormat,
outputBounds=bounds,
outputType=gdal.GDT_Byte,
width=arrshape[1],
height=arrshape[0],
multithread=True,
options=['NUM_THREADS=%s' % (num_threads)]))
update_file = gdal.Open(maskfilename, gdal.GA_Update)
update_file.SetProjection(proj)
update_file.GetRasterBand(1).SetNoDataValue(0.)
del update_file
gdal.Translate(maskfilename + '.vrt',
maskfilename,
options=gdal.TranslateOptions(format="VRT"))
###Must take inverse of lakes/ponds union because of opposite designation (1 for water, 0 for land) as desired (0 for water, 1 for land)
lake_masks = gdal.Rasterize(
'',
os.path.join(workdir, 'watermsk_lakes.vrt'),
options=gdal.RasterizeOptions(format='MEM',
outputBounds=bounds,
outputType=gdal.GDT_Byte,
width=arrshape[1],
height=arrshape[0],
burnValues=[1],
layers='merged',
inverse=True))
lake_masks.SetProjection(proj)
lake_masks = lake_masks.ReadAsArray()
###Update water-mask with lakes/ponds union and average amplitude
mask_file = gdal.Open(maskfilename, gdal.GA_Update)
mask_file.GetRasterBand(1).WriteArray(
lake_masks * gdal.Open(maskfilename).ReadAsArray())
#Delete temp files
del lake_masks, mask_file
if os.path.basename(maskfilename).lower().startswith('nlcd'):
log.info("***Accessing and cropping the NLCD mask...***")
maskfilename = NLCDMasker(os.path.dirname(workdir))(proj, bounds,
arrshape,
outputFormat)
# Make sure to apply amplitude mask to downloaded products
if os.path.exists(os.path.join(workdir, 'watermsk_shorelines.vrt')
) or os.path.basename(maskfilename) == 'NLCD_crop.msk':
###Make average amplitude mask
if amp_thresh:
amp_file = rasterAverage(os.path.join(workdir, 'avgamplitude'),
product_dict[0],
bounds,
prods_TOTbbox,
outputFormat=outputFormat,
thresh=amp_thresh)
###Update mask with average amplitude
mask_file = gdal.Open(maskfilename, gdal.GA_Update)
mask_file.GetRasterBand(1).WriteArray(
gdal.Open(maskfilename).ReadAsArray() * amp_file)
#Delete temp files
del mask_file, amp_file
if os.path.exists(os.path.join(workdir, 'watermsk_shorelines.vrt')):
os.remove(os.path.join(workdir, 'watermsk_shorelines.vrt'))
if os.path.exists(os.path.join(workdir, 'watermsk_lakes.vrt')):
os.remove(os.path.join(workdir, 'watermsk_lakes.vrt'))
# Load mask
try:
# Check if uncropped/cropped maskfiles exist in 'mask' subdirectory
if not os.path.exists(
os.path.join(
workdir,
os.path.basename(maskfilename).split('.')[0] + '.msk')):
# save uncropped masfile
gdal.BuildVRT(os.path.join(
workdir,
os.path.basename(maskfilename).split('.')[0] +
'_uncropped.msk.vrt'),
maskfilename,
options=gdal.BuildVRTOptions(outputBounds=bounds))
# update maskfilename
maskfilename = os.path.join(
workdir,
os.path.basename(maskfilename).split('.')[0].split('uncropped')
[0] + '.msk')
# save cropped maskfile
gdal.Warp(maskfilename,
os.path.join(
workdir,
os.path.basename(maskfilename).split('.')[0] +
'_uncropped.msk.vrt'),
options=gdal.WarpOptions(
format=outputFormat,
cutlineDSName=prods_TOTbbox,
outputBounds=bounds,
width=arrshape[1],
height=arrshape[0],
multithread=True,
options=['NUM_THREADS=%s' % (num_threads)]))
mask_file = gdal.Open(maskfilename, gdal.GA_Update)
mask_file.SetProjection(proj)
del mask_file
gdal.Translate(maskfilename + '.vrt',
maskfilename,
options=gdal.TranslateOptions(format="VRT"))
###Make average amplitude mask
if amp_thresh:
amp_file = rasterAverage(os.path.join(workdir, 'avgamplitude'),
product_dict[0],
bounds,
prods_TOTbbox,
outputFormat=outputFormat,
thresh=amp_thresh)
###Update mask with average amplitude
mask_file = gdal.Open(maskfilename, gdal.GA_Update)
mask_file.GetRasterBand(1).WriteArray(
gdal.Open(maskfilename).ReadAsArray() * amp_file)
#Delete temp files
del mask_file, amp_file
#pass cropped DEM
mask = gdal.Warp('',
maskfilename,
options=gdal.WarpOptions(
format="MEM",
cutlineDSName=prods_TOTbbox,
outputBounds=bounds,
width=arrshape[1],
height=arrshape[0],
multithread=True,
options=['NUM_THREADS=%s' % (num_threads)]))
mask.SetProjection(proj)
mask.SetDescription(maskfilename)
except:
raise Exception('Failed to open user mask')
return mask
def __continuous_time__(self):
'''
Split the products into spatiotemporally continuous groups (i.e. by individual, continuous interferograms). Input must be already sorted by pair and start-time to fit the logic scheme below.
Using their time-tags, this function determines whether or not successive products are in the same orbit. If in the same orbit, the program determines whether or not they overlap in time and are therefore spatially contiguous, and rejects/reports cases for which there is no temporal overlap and therefore a spatial gap.
'''
# import dependencies
from datetime import datetime, timedelta
import itertools
sorted_products = []
track_rejected_pairs = []
# Check for (and remove) duplicate products
num_prods = len(self.products)
num_dups = []
for i in enumerate(self.products[:-1]):
# If scenes share >90% spatial overlap AND same dates, they MUST be duplicates. Reject the latter.
if (self.products[i[0] + 1][0]['pair_name'][9:]
== i[1][0]['pair_name'][9:]) and (
self.products[i[0] + 1][0]['pair_name'][:8]
== i[1][0]['pair_name'][:8]) and (open_shapefile(
self.products[i[0] + 1][1]['productBoundingBox'],
'productBoundingBox', 1).intersection(
open_shapefile(i[1][1]['productBoundingBox'],
'productBoundingBox', 1)
).area) / (open_shapefile(
i[1][1]['productBoundingBox'],
'productBoundingBox', 1).area) > 0.9:
log.debug(
"Duplicate product captured. Rejecting scene %s",
os.path.basename(
self.products[i[0] +
1][1]['unwrappedPhase'].split('"')[1]))
# Overwrite latter scene with former
self.products[i[0] + 1] = i[1]
num_dups.append(i[0])
# Delete duplicate products
self.products = list(
self.products
for self.products, _ in itertools.groupby(self.products))
if num_dups:
log.warning("%d products rejected since they are duplicates",
len(num_dups))
# If only one pair in list, add it to list.
if len(self.products) == 1:
sorted_products.extend([[
dict(
zip(self.products[0][0].keys(),
[list(a) for a in zip(self.products[0][0].values())])),
dict(
zip(self.products[0][1].keys(),
[list(a) for a in zip(self.products[0][1].values())]))
]])
# If multiple pairs in list, cycle through and evaluate temporal connectivity.
for i in enumerate(self.products[:-1]):
# Get this reference product's times
scene_start = datetime.strptime(
i[1][0]['azimuthZeroDopplerMidTime'], "%Y-%m-%dT%H:%M:%S.%f")
scene_end = scene_start + timedelta(seconds=i[1][0]['sceneLength'])
master = datetime.strptime(i[1][0]['pair_name'][9:], "%Y%m%d")
new_scene_start = datetime.strptime(
self.products[i[0] + 1][0]['azimuthZeroDopplerMidTime'],
"%Y-%m-%dT%H:%M:%S.%f")
new_scene_end = new_scene_start + timedelta(
seconds=i[1][0]['sceneLength'])
slave = datetime.strptime(
self.products[i[0] + 1][0]['pair_name'][9:], "%Y%m%d")
# Determine if next product in time is in same orbit AND overlaps AND corresponds to same pair
# If it is within same orbit cycle, try to append scene. This accounts for day change.
if abs(new_scene_end - scene_end) <= timedelta(
minutes=100) and abs(slave - master) <= timedelta(days=1):
# Don't export product if it is already tracked as a rejected pair
if i[1][0][
'pair_name'] in track_rejected_pairs or self.products[
i[0] + 1][0]['pair_name'] in track_rejected_pairs:
track_rejected_pairs.extend(
(i[1][0]['pair_name'],
self.products[i[0] + 1][0]['pair_name']))
# Only pass scene if it temporally overlaps with reference scene
elif ((scene_end <= new_scene_start) and
(new_scene_end <= scene_start)) or (
(scene_end >= new_scene_start) and
(new_scene_end >= scene_start)):
# Check if dictionary for IFG corresponding to reference product already exists, and if it does then append values
try:
dict_ind = sorted_products.index(
next(item for item in sorted_products
if i[1][1]['productBoundingBox'] in item[1]
['productBoundingBox']))
sorted_products[dict_ind] = [
dict(
zip(i[1][0].keys(), [[
subitem for item in a
for subitem in (item if isinstance(
item, list) else [item])
] for a in zip(
sorted_products[dict_ind][0].values(),
self.products[i[0] + 1][0].values())])),
dict(
zip(i[1][1].keys(), [[
subitem for item in a
for subitem in (item if isinstance(
item, list) else [item])
] for a in zip(
sorted_products[dict_ind][1].values(),
self.products[i[0] + 1][1].values())]))
]
# Match IFG corresponding to reference product NOT found, so initialize dictionary for new IFG
except:
sorted_products.extend([[
dict(
zip(i[1][0].keys(), [
list(a) for a in zip(
i[1][0].values(), self.products[
i[0] + 1][0].values())
])),
dict(
zip(i[1][1].keys(), [
list(a) for a in zip(
i[1][1].values(), self.products[
i[0] + 1][1].values())
]))
]])
#Else if scene doesn't overlap, this means there is a gap. Reject date from product list, and keep track of all failed dates
else:
track_rejected_pairs.extend(
(i[1][0]['pair_name'],
self.products[i[0] + 1][0]['pair_name']))
log.debug("Gap for interferogram %s", i[1][0]['pair_name'])
# Products correspond to different dates, so pass both as separate IFGs.
else:
# Check if dictionary for IFG corresponding to reference product already exists. If not, then pass as new IFG.
if [
item for item in sorted_products if i[1][1]
['productBoundingBox'] in item[1]['productBoundingBox']
] == [] and i[1][0]['pair_name'] not in track_rejected_pairs:
sorted_products.extend([[
dict(
zip(i[1][0].keys(),
[list(a) for a in zip(i[1][0].values())])),
dict(
zip(i[1][1].keys(),
[list(a) for a in zip(i[1][1].values())]))
]])
# Check if dictionary for IFG corresponding to next product already exists. If not, then pass as new IFG.
if [
item
for item in sorted_products if self.products[i[0] + 1]
[1]['productBoundingBox'] in item[1]['productBoundingBox']
] == [] and self.products[
i[0] + 1][0]['pair_name'] not in track_rejected_pairs:
sorted_products.extend([[
dict(
zip(self.products[i[0] + 1][0].keys(), [
list(a)
for a in zip(self.products[i[0] +
1][0].values())
])),
dict(
zip(self.products[i[0] + 1][1].keys(), [
list(a)
for a in zip(self.products[i[0] +
1][1].values())
]))
]])
# Remove duplicate dates
track_rejected_pairs = list(set(track_rejected_pairs))
if len(track_rejected_pairs) > 0:
log.warning(
"%d out of %d interferograms rejected since stitched interferogram would have gaps",
len(track_rejected_pairs),
len([item[0] for item in sorted_products]))
# Provide report of which files were kept vs. which were not.
log.debug(
"Specifically, the following interferograms were rejected:")
for item in sorted_products:
if item[1]['pair_name'][0] in track_rejected_pairs:
log.debug(
str([
rejects.split('"')[1]
for rejects in item[1]['productBoundingBox']
]).strip('[]'))
else:
log.info("All (%d) interferograms are spatially continuous.",
len(sorted_products))
sorted_products=[[item[0] for item in sorted_products if (item[0]['pair_name'][0] not in track_rejected_pairs)], \
[item[1] for item in sorted_products if (item[1]['pair_name'][0] not in track_rejected_pairs)]]
###Report dictionaries for all valid products
if sorted_products == [[],
[]]: #Check if pairs were successfully selected
raise Exception(
'No valid interferogram meet spatial criteria due to gaps and/or invalid input, nothing to export.'
)
return sorted_products
def main(inps=None):
inps = cmdLineParse()
print("***Time-series Preparation Function:***")
# if user bbox was specified, file(s) not meeting imposed spatial criteria are rejected.
# Outputs = arrays ['standardproduct_info.products'] containing grouped “radarmetadata info” and “data layer keys+paths” dictionaries for each standard product
# In addition, path to bbox file ['standardproduct_info.bbox_file'] (if bbox specified)
standardproduct_info = ARIA_standardproduct(inps.imgfile, bbox=inps.bbox, workdir=inps.workdir, verbose=inps.verbose)
# pass number of threads for gdal multiprocessing computation
if inps.num_threads.lower()=='all':
import multiprocessing
print('User specified use of all %s threads for gdal multiprocessing'%(str(multiprocessing.cpu_count())))
inps.num_threads='ALL_CPUS'
print('Thread count specified for gdal multiprocessing = %s'%(inps.num_threads))
# extract/merge productBoundingBox layers for each pair and update dict,
# report common track bbox (default is to take common intersection, but user may specify union), and expected shape for DEM.
standardproduct_info.products[0], standardproduct_info.products[1], standardproduct_info.bbox_file, prods_TOTbbox, prods_TOTbbox_metadatalyr, arrshape, proj = merged_productbbox(standardproduct_info.products[0], standardproduct_info.products[1], os.path.join(inps.workdir,'productBoundingBox'), standardproduct_info.bbox_file, inps.croptounion, num_threads=inps.num_threads, minimumOverlap=inps.minimumOverlap, verbose=inps.verbose)
# Download/Load DEM & Lat/Lon arrays, providing bbox, expected DEM shape, and output dir as input.
if inps.demfile is not None:
print('Download/cropping DEM')
# Pass DEM-filename, loaded DEM array, and lat/lon arrays
inps.demfile, demfile, Latitude, Longitude = prep_dem(inps.demfile, standardproduct_info.bbox_file, prods_TOTbbox, \
prods_TOTbbox_metadatalyr, proj, arrshape=arrshape, workdir=inps.workdir, outputFormat=inps.outputFormat, num_threads=inps.num_threads)
# Load or download mask (if specified).
if inps.mask is not None:
inps.mask = prep_mask([[item for sublist in [list(set(d['amplitude'])) for d in standardproduct_info.products[1] if 'amplitude' in d] for item in sublist], [item for sublist in [list(set(d['pair_name'])) for d in standardproduct_info.products[1] if 'pair_name' in d] for item in sublist]],inps.mask, standardproduct_info.bbox_file, prods_TOTbbox, proj, amp_thresh=inps.amp_thresh, arrshape=arrshape, workdir=inps.workdir, outputFormat=inps.outputFormat, num_threads=inps.num_threads)
# Extract
layers=['unwrappedPhase','coherence']
print('\nExtracting unwrapped phase, coherence, and connected components for each interferogram pair')
export_products(standardproduct_info.products[1], standardproduct_info.bbox_file, prods_TOTbbox, layers, dem=demfile, lat=Latitude, lon=Longitude, mask=inps.mask, outDir=inps.workdir, outputFormat=inps.outputFormat, stitchMethodType='overlap', verbose=inps.verbose, num_threads=inps.num_threads, multilooking=inps.multilooking)
layers=['incidenceAngle','lookAngle','azimuthAngle']
print('\nExtracting single incidence angle, look angle and azimuth angle files valid over common interferometric grid')
export_products([dict(zip([k for k in set(k for d in standardproduct_info.products[1] for k in d)], [[item for sublist in [list(set(d[k])) for d in standardproduct_info.products[1] if k in d] for item in sublist] for k in set(k for d in standardproduct_info.products[1] for k in d)]))], standardproduct_info.bbox_file, prods_TOTbbox, layers, dem=demfile, lat=Latitude, lon=Longitude, mask=inps.mask, outDir=inps.workdir, outputFormat=inps.outputFormat, stitchMethodType='overlap', verbose=inps.verbose, num_threads=inps.num_threads, multilooking=inps.multilooking)
if inps.bperp==True:
layers=['bPerpendicular']
print('\nExtracting perpendicular baseline grids for each interferogram pair')
export_products(standardproduct_info.products[1], standardproduct_info.bbox_file, prods_TOTbbox, layers, dem=demfile, lat=Latitude, lon=Longitude, mask=inps.mask, outDir=inps.workdir, outputFormat=inps.outputFormat, stitchMethodType='overlap', verbose=inps.verbose, num_threads=inps.num_threads, multilooking=inps.multilooking)
# Extracting other layers, if specified
if inps.layers:
if inps.layers.lower()=='all':
inps.layers=list(standardproduct_info.products[1][0].keys())
# Must also remove productBoundingBoxes & pair-names because they are not raster layers
layers=[i for i in inps.layers if i not in ['unwrappedPhase','coherence','incidenceAngle','lookAngle','azimuthAngle','bPerpendicular','productBoundingBox','productBoundingBoxFrames','pair_name']]
else:
inps.layers=list(inps.layers.split(','))
layers=[i for i in inps.layers if i not in ['unwrappedPhase','coherence','incidenceAngle','lookAngle','azimuthAngle','bPerpendicular']]
layers=[i.replace(' ','') for i in layers]
if layers!=[]:
print('\nExtracting optional, user-specified layers %s for each interferogram pair'%(layers))
export_products(standardproduct_info.products[1], standardproduct_info.bbox_file, prods_TOTbbox, layers, dem=demfile, lat=Latitude, lon=Longitude, mask=inps.mask, outDir=inps.workdir, outputFormat=inps.outputFormat, stitchMethodType='overlap', verbose=inps.verbose, num_threads=inps.num_threads, multilooking=inps.multilooking)
# If necessary, resample DEM/mask AFTER they have been used to extract metadata layers and mask output layers, respectively
if inps.multilooking is not None:
# Import functions
from ARIAtools.shapefile_util import open_shapefile
from ARIAtools.vrtmanager import resampleRaster
bounds=open_shapefile(standardproduct_info.bbox_file, 0, 0).bounds
# Resample mask
if inps.mask is not None:
resampleRaster(inps.mask.GetDescription(), inps.multilooking, bounds, prods_TOTbbox, inps.rankedResampling, outputFormat=inps.outputFormat, num_threads=inps.num_threads)
# Resample DEM
if demfile is not None:
resampleRaster(demfile.GetDescription(), inps.multilooking, bounds, prods_TOTbbox, inps.rankedResampling, outputFormat=inps.outputFormat, num_threads=inps.num_threads)
# Perform GACOS-based tropospheric corrections (if specified).
if inps.tropo_products:
tropo_correction(standardproduct_info.products, inps.tropo_products, standardproduct_info.bbox_file, prods_TOTbbox, outDir=inps.workdir, outputFormat=inps.outputFormat, verbose=inps.verbose, num_threads=inps.num_threads)
# Generate Stack
generateStack(standardproduct_info,'coherence','cohStack',workdir=inps.workdir)
generateStack(standardproduct_info,'connectedComponents','connCompStack',workdir=inps.workdir)
if inps.tropo_products:
generateStack(standardproduct_info,'tropocorrected_products','unwrapStack',workdir=inps.workdir)
else:
generateStack(standardproduct_info,'unwrappedPhase','unwrapStack',workdir=inps.workdir)
def prep_mask(product_dict,
maskfilename,
bbox_file,
prods_TOTbbox,
proj,
amp_thresh=None,
arrshape=None,
workdir='./',
outputFormat='ENVI',
num_threads='2'):
"""Function to load and export mask file.
If "Download" flag, GSHHS water mask will be donwloaded on the fly.
If full resolution NLCD landcover data is given (NLCD...img) it gets cropped
"""
# Import functions
from ARIAtools.vrtmanager import renderOGRVRT
_world_watermask = [f' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/'\
f'gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L{i}.shp'\
for i in range(1, 5)]
_world_watermask.append( ' /vsizip/vsicurl/https://osmdata.openstreetmap.de'\
'/download/land-polygons-complete-4326.zip/'\
'land-polygons-complete-4326/land_polygons.shp')
# If specified DEM subdirectory exists, delete contents
workdir = os.path.join(workdir, 'mask')
os.makedirs(workdir, exist_ok=True)
# Get bounds of user bbox_file
bounds = open_shapefile(bbox_file, 0, 0).bounds
# File must be physically extracted, cannot proceed with VRT format. Defaulting to ENVI format.
if outputFormat == 'VRT':
outputFormat = 'ENVI'
# Download mask
if maskfilename.lower() == 'download':
log.info("***Downloading water mask... ***")
maskfilename = os.path.join(workdir, 'watermask' + '.msk')
os.environ['CPL_ZIP_ENCODING'] = 'UTF-8'
###Make coastlines/islands union VRT
renderOGRVRT(os.path.join(workdir, 'watermsk_shorelines.vrt'),
_world_watermask[::2])
###Make lakes/ponds union VRT
renderOGRVRT(os.path.join(workdir, 'watermsk_lakes.vrt'),
_world_watermask[1::2])
###Initiate water-mask with coastlines/islands union VRT
# save uncropped mask
gdal.Rasterize(os.path.join(workdir, 'watermask_uncropped.msk'),
os.path.join(workdir, 'watermsk_shorelines.vrt'),
format=outputFormat,
outputBounds=bounds,
outputType=gdal.GDT_Byte,
width=arrshape[1],
height=arrshape[0],
burnValues=[1],
layers='merged')
gdal.Translate(os.path.join(workdir, 'watermask_uncropped.msk.vrt'),
os.path.join(workdir, 'watermask_uncropped.msk'),
format="VRT")
# save cropped mask
gdal.Warp(maskfilename,
os.path.join(workdir, 'watermask_uncropped.msk.vrt'),
format=outputFormat,
outputBounds=bounds,
outputType=gdal.GDT_Byte,
width=arrshape[1],
height=arrshape[0],
multithread=True,
options=['NUM_THREADS=%s' % (num_threads)])
update_file = gdal.Open(maskfilename, gdal.GA_Update)
update_file.SetProjection(proj)
update_file.GetRasterBand(1).SetNoDataValue(0.)
del update_file
gdal.Translate(f'{maskfilename}.vrt', maskfilename, format='VRT')
###Must take inverse of lakes/ponds union because of opposite designation (1 for water, 0 for land) as desired (0 for water, 1 for land)
lake_masks = gdal.Rasterize('',
os.path.join(workdir,
'watermsk_lakes.vrt'),
format='MEM',
outputBounds=bounds,
outputType=gdal.GDT_Byte,
width=arrshape[1],
height=arrshape[0],
burnValues=[1],
layers='merged',
inverse=True)
lake_masks.SetProjection(proj)
lake_masks = lake_masks.ReadAsArray()
## Update water-mask with lakes/ponds union
mask_file = gdal.Open(maskfilename, gdal.GA_Update)
mask_file.GetRasterBand(1).WriteArray(
lake_masks * gdal.Open(maskfilename).ReadAsArray())
#Delete temp files
del lake_masks, mask_file
## Use NLCD Mask
elif os.path.basename(maskfilename).lower().startswith('nlcd'):
log.info("***Accessing and cropping the NLCD mask...***")
maskfilename = NLCDMasker(os.path.dirname(workdir))(proj, bounds,
arrshape,
outputFormat)
## User specified mask
else:
# Path to local version of user specified mask
user_mask = maskfilename # for clarity
user_mask_n = os.path.basename(os.path.splitext(user_mask)[0])
local_mask = os.path.join(workdir, f'{user_mask_n}.msk')
local_mask_unc = os.path.join(workdir, f'{user_mask_n}_uncropped.msk')
if user_mask == local_mask:
log.warning('The mask you specified already exists in %s, '\
'using the existing one...', os.path.dirname(local_mask))
else:
# move the mask to the local directory and built a VRT for it
gdal.UseExceptions()
# shutil.copy(user_mask, local_mask_unc)
ds = gdal.BuildVRT(f'{local_mask_unc}.vrt',
user_mask,
outputBounds=bounds)
assert ds is not None, f'GDAL could not open user mask: {user_mask}'
# crop the user mask and write
gdal.Warp(f'{local_mask}',
ds,
format=outputFormat,
cutlineDSName=prods_TOTbbox,
outputBounds=bounds,
width=arrshape[1],
height=arrshape[0],
multithread=True,
options=[f'NUM_THREADS={num_threads}'])
# set projection of the local mask
mask_file = gdal.Open(local_mask, gdal.GA_Update)
mask_file.SetProjection(proj)
del mask_file
# create vrt for local cropped mask
gdal.Translate(f'{local_mask}.vrt', local_mask, format='VRT')
# assign new local mask for amp thresh
maskfilename = local_mask
## Make average amplitude mask
if amp_thresh is not None:
amp_file = rasterAverage(os.path.join(workdir, 'avgamplitude'),
product_dict,
bounds,
prods_TOTbbox,
outputFormat=outputFormat,
thresh=amp_thresh)
# Update mask with average amplitude
mask_file = gdal.Open(maskfilename, gdal.GA_Update)
mask_arr = mask_file.ReadAsArray()
mask_file.GetRasterBand(1).WriteArray(mask_arr * amp_file)
# Delete temp files
del mask_file, amp_file
# crop/expand mask to DEM size?
mask = gdal.Warp('',
maskfilename,
format='MEM',
cutlineDSName=prods_TOTbbox,
outputBounds=bounds,
width=arrshape[1],
height=arrshape[0],
multithread=True,
options=[f'NUM_THREADS={num_threads}'])
mask.SetProjection(proj)
mask.SetDescription(maskfilename)
try:
## remove extra files
os.remove(path_shorelines) if os.path.exists(path_shorelines) else ''
os.remove(path_lakes) if os.path.exists(path_lakes) else ''
except:
pass
return mask
