def reproject_stripmeta(metafile_src, metafile_dst, target_epsg, args): outmeta_txt = "" inmeta_fp = open(metafile_src, 'r') target_res = args.get(ARGSTR_TARGET_RESOLUTION) # Create output spatial reference and get # output projection in proj4 format. spatref_out = osr_srs_preserve_axis_order(osr.SpatialReference()) spatref_out.ImportFromEPSG(target_epsg) proj4_out = spatref_out.ExportToProj4() # Get strip projection. line = inmeta_fp.readline() while not line.startswith('Strip projection (proj4):') and line != "": outmeta_txt += line line = inmeta_fp.readline() if line == "": logger.error( "Projection string cannot be parsed from meta file: {}".format( metafile_src)) inmeta_fp.close() return proj4_in = line.split("'")[1] outmeta_txt += "Strip projection (proj4): '{}'\n".format(proj4_out) # Get strip footprint geometry. line = inmeta_fp.readline() while not line.startswith('Strip Footprint Vertices') and line != "": outmeta_txt += line line = inmeta_fp.readline() if line == "": logger.error( "Footprint vertices cannot be parsed from meta file: {}".format( metafile_src)) inmeta_fp.close() return outmeta_txt += line if args.get(ARGSTR_CORRECT_META_FP_VERTS): ## Recalculate strip footprint vertices from reprojected strip DEM raster. from lib import raster_array_tools as rat for i in range(2): inmeta_fp.readline() demfile_dst = metafile_dst.replace(SETSM_META_SUFFIX, STRIP_DEM_SUFFIX) Z, Y, X = raster_io.extractRasterData(demfile_dst, 'z', 'y', 'x') fp_vertices = rat.getFPvertices(Z, X, Y, label=-9999, label_type='nodata', replicate_matlab=True, dtype_out_int64_if_equal=True) del Z, Y, X x_out, y_out = fp_vertices else: ## Reproject existing strip footprint vertices. line = inmeta_fp.readline() x_in = np.fromstring(line.replace('X:', '').strip(), dtype=np.float64, sep=' ') line = inmeta_fp.readline() y_in = np.fromstring(line.replace('Y:', '').strip(), dtype=np.float64, sep=' ') wkt_in = raster_io.coordsToWkt(np.array([x_in, y_in]).T) # Create input footprint geometry with spatial reference. geom = ogr.Geometry(wkt=wkt_in) spatref_in = osr_srs_preserve_axis_order(osr.SpatialReference()) spatref_in.ImportFromProj4(proj4_in) geom.AssignSpatialReference(spatref_in) # Transform geometry to new spatial reference # and extract transformed coordinates. geom.TransformTo(spatref_out) wkt_out = geom.ExportToWkt() x_out, y_out = raster_io.wktToCoords(wkt_out).T # x_out = (np.around(x_out / target_res) * target_res).astype(np.int64) # y_out = (np.around(y_out / target_res) * target_res).astype(np.int64) outmeta_txt += "X: {}\n".format(' '.join( np.array_str(x_out, max_line_width=np.inf).strip()[1:-1].split())) outmeta_txt += "Y: {}\n".format(' '.join( np.array_str(y_out, max_line_width=np.inf).strip()[1:-1].split())) outmeta_txt += inmeta_fp.read() inmeta_fp.close() # Write output metadata file. outmeta_fp = open(metafile_dst, 'w') outmeta_fp.write(outmeta_txt) outmeta_fp.close()
def reproject_setsm(src_metafile, dstdir=None, target_epsg=None, target_resolution=None, args=None): src_is_strip_metafile = DRE_STRIPMETAFILE.fullmatch( os.path.basename(src_metafile)).matched normal_strip_output = (target_resolution in (None, 2) and not args.get(ARGSTR_ADD_RES_SUFFIX)) if args is None and any([a is None for a in [dstdir, target_epsg]]): raise script_utils.InvalidArgumentError( "`args` must be provided if `dstdir` or `target_epsg` is None") if dstdir is None: dstdir = args.get(ARGSTR_DSTDIR) if target_epsg is None: target_epsg = args.get(ARGSTR_TARGET_EPSG) if target_resolution is None: target_resolution = args.get(ARGSTR_TARGET_RESOLUTION) if not os.path.isfile(src_metafile): raise script_utils.InvalidArgumentError( "Source metafile does not exist: {}".format(src_metafile)) dstdir = get_task_dstdir(src_metafile, dstdir) if not os.path.isdir(dstdir): if not args.get(ARGSTR_DRYRUN): os.makedirs(dstdir, exist_ok=True) if src_is_strip_metafile and normal_strip_output: src_rasters_to_reproject = [ src_metafile.replace(SETSM_META_SUFFIX, suffix) for suffix in BUILD_STRIP_AUX_CORE_STRIP_SUFFIXES ] src_ortho2file = src_metafile.replace(SETSM_META_SUFFIX, 'ortho2.tif') if os.path.isfile(src_ortho2file): src_rasters_to_reproject.append(src_ortho2file) else: src_rasters_to_reproject = glob.glob( src_metafile.replace(SETSM_META_SUFFIX, '*.tif')) commands = [] for rasterfile_src in src_rasters_to_reproject: rasterfile_dst = os.path.join(dstdir, os.path.basename(rasterfile_src)) rasterfile_dst_uncompressed = None if args.get(ARGSTR_ADD_RES_SUFFIX): rasterfile_dst_base, rasterfile_dst_ext = os.path.splitext( rasterfile_dst) rasterfile_dst = "{}_{}m{}".format( rasterfile_dst_base, args.get(ARGSTR_TARGET_RESOLUTION), rasterfile_dst_ext) rasterfile_dst_res = (target_resolution if target_resolution is not None else raster_io.extractRasterData( rasterfile_src, 'res')) if rasterfile_src.endswith('dem.tif'): rasterfile_dst_base, rasterfile_dst_ext = os.path.splitext( rasterfile_dst) rasterfile_dst_uncompressed = "{}_uncompressed{}".format( rasterfile_dst_base, rasterfile_dst_ext) resample_method = None for suffix_key, suffix_method in SETSM_SUFFIX_KEY_TO_RESAMPLE_METHOD_LOOKUP: if suffix_key in rasterfile_src: resample_method = suffix_method break if resample_method is None: raise script_utils.DeveloperError( "No suffix key in `SETSM_SUFFIX_KEY_TO_RESAMPLE_METHOD_LOOKUP` matches " "source raster file: {}".format(rasterfile_src)) if rasterfile_src.endswith(('ortho.tif', 'ortho2.tif')): predictor = 2 else: predictor = 1 commands.append(( 'gdalwarp "{0}" "{1}" -q -tap -t_srs EPSG:{2} -tr {3} {3} -r {4} {5} ' '-overwrite -co TILED=YES -co BIGTIFF=YES -co COMPRESS=LZW -co PREDICTOR={6}' .format( rasterfile_src, rasterfile_dst if rasterfile_dst_uncompressed is None else rasterfile_dst_uncompressed, target_epsg, rasterfile_dst_res, resample_method, '-dstnodata 1' if 'bitmask' in rasterfile_src else '', predictor, ))) if rasterfile_src.endswith('dem.tif'): commands.append(( 'gdal_calc.py --quiet --overwrite -A "{0}" --outfile="{1}" --calc="round_(A*128.0)/128.0" --NoDataValue=-9999' ' --co TILED=YES --co BIGTIFF=YES --co COMPRESS=LZW --co PREDICTOR=3' .format(rasterfile_dst_uncompressed, rasterfile_dst))) elif 'bitmask' in rasterfile_src: commands.append( 'gdal_edit.py -unsetnodata {}'.format(rasterfile_dst)) for cmd in commands: print(cmd) if not args.get(ARGSTR_DRYRUN): rc, _, _ = script_utils.exec_cmd(cmd) if rc != 0: raise script_utils.ExternalError( "External program call finished with non-zero exit status ({})" .format(rc)) metafile_src = src_metafile metafile_dst = os.path.join(dstdir, os.path.basename(metafile_src)) if src_is_strip_metafile and normal_strip_output: demfile_dst = metafile_dst.replace(SETSM_META_SUFFIX, STRIP_DEM_SUFFIX) saveStripBrowse(args, demfile_dst, STRIP_DEM_SUFFIX, 'bitmask.tif') if src_is_strip_metafile: print( "Reprojecting strip footprint vertices in metafile to EPSG:{}: {} -> {}" .format(target_epsg, metafile_src, metafile_dst)) if not args.get(ARGSTR_DRYRUN): reproject_stripmeta(metafile_src, metafile_dst, target_epsg, args) else: print("Simply copying scene metafile: {} -> {}".format( metafile_src, metafile_dst)) if not args.get(ARGSTR_DRYRUN): shutil.copyfile(metafile_src, metafile_dst)
def reproject_setsm(src_metafile, dstdir=None, target_epsg=None, target_resolution=None, args=None): src_is_strip_metafile = demregex.StripDemFile( os.path.basename(src_metafile)).match().matched normal_strip_output = (target_resolution in (None, 2) and not args.get(ARGSTR_ADD_RES_SUFFIX)) if args is None and any([a is None for a in [dstdir, target_epsg]]): raise script_utils.InvalidArgumentError( "`args` must be provided if `dstdir` or `target_epsg` is None") if dstdir is None: dstdir = args.get(ARGSTR_DSTDIR) if target_epsg is None: target_epsg = args.get(ARGSTR_TARGET_EPSG) if target_resolution is None: target_resolution = args.get(ARGSTR_TARGET_RESOLUTION) if not os.path.isfile(src_metafile): raise script_utils.InvalidArgumentError( "Source metafile does not exist: {}".format(src_metafile)) dstdir = get_task_dstdir(src_metafile, dstdir) if not os.path.isdir(dstdir): if not args.get(ARGSTR_DRYRUN): os.makedirs(dstdir, exist_ok=True) if src_is_strip_metafile and normal_strip_output: src_rasters_to_reproject = [ src_metafile.replace(SETSM_META_SUFFIX, suffix) for suffix in BUILD_STRIP_AUX_CORE_STRIP_SUFFIXES ] src_ortho2file = src_metafile.replace(SETSM_META_SUFFIX, 'ortho2.tif') if os.path.isfile(src_ortho2file): src_rasters_to_reproject.append(src_ortho2file) else: src_rasters_to_reproject = glob.glob( src_metafile.replace(SETSM_META_SUFFIX, '*.tif')) commands = [] for rasterfile_src in src_rasters_to_reproject: rasterfile_dst = os.path.join(dstdir, os.path.basename(rasterfile_src)) if args.get(ARGSTR_ADD_RES_SUFFIX): rasterfile_dst_base, rasterfile_dst_ext = os.path.splitext( rasterfile_dst) rasterfile_dst = "{}_{}m{}".format( rasterfile_dst_base, args.get(ARGSTR_TARGET_RESOLUTION), rasterfile_dst_ext) rasterfile_dst_res = (target_resolution if target_resolution is not None else raster_io.extractRasterData( rasterfile_src, 'res')) resample_method = None for suffix_key, suffix_method in SETSM_SUFFIX_KEY_TO_RESAMPLE_METHOD_LOOKUP: if suffix_key in rasterfile_src: resample_method = suffix_method break if resample_method is None: raise script_utils.DeveloperError( "No suffix key in `SETSM_SUFFIX_KEY_TO_RESAMPLE_METHOD_LOOKUP` matches " "source raster file: {}".format(rasterfile_src)) commands.append(( 'gdalwarp "{0}" "{1}" -q -tap -t_srs EPSG:{2} -tr {3} {3} -r {4} {5} ' '-overwrite -co TILED=YES -co BIGTIFF=IF_SAFER -co COMPRESS=LZW'. format(rasterfile_src, rasterfile_dst, target_epsg, rasterfile_dst_res, resample_method, '-dstnodata 1' if 'bitmask' in rasterfile_src else ''))) if 'bitmask' in rasterfile_src: commands.append( 'gdal_edit.py -unsetnodata {}'.format(rasterfile_dst)) for cmd in commands: print(cmd) if not args.get(ARGSTR_DRYRUN): script_utils.exec_cmd(cmd) metafile_src = src_metafile metafile_dst = os.path.join(dstdir, os.path.basename(metafile_src)) if src_is_strip_metafile and normal_strip_output: demfile_dst = metafile_dst.replace(SETSM_META_SUFFIX, STRIP_DEM_SUFFIX) saveStripBrowse(args, demfile_dst, STRIP_DEM_SUFFIX, 'bitmask.tif') if src_is_strip_metafile: print( "Reprojecting strip footprint vertices in metafile to EPSG:{}: {} -> {}" .format(target_epsg, metafile_src, metafile_dst)) if not args.get(ARGSTR_DRYRUN): reproject_stripmeta(metafile_src, metafile_dst, target_epsg, args) else: print("Simply copying scene metafile: {} -> {}".format( metafile_src, metafile_dst)) if not args.get(ARGSTR_DRYRUN): shutil.copyfile(metafile_src, metafile_dst)
def mask_rasters(maskFile, suffix_maskval_dict, args): import numpy as np import lib.raster_io as rio global SRC_SUFFIX_CATCH_ALL nodata_opt = args.get(ARGSTR_DST_NODATA) bitmaskSuffix = getBitmaskSuffix(maskFile) bitmask_is_strip_lsf = (bitmaskSuffix == SUFFIX_STRIP_BITMASK_LSF) if suffix_maskval_dict is None: # maskFile_base = maskFile.replace(bitmaskSuffix, '') # suffix_maskval_dict = {src_rasterFile.replace(maskFile_base, ''): None # for src_rasterFile in glob.glob(maskFile_base+SRC_SUFFIX_CATCH_ALL) # if not src_rasterFile.endswith(bitmaskSuffix)} suffix_maskval_dict = {SRC_SUFFIX_CATCH_ALL: None} if True in args.get(ARGSTR_EDGE, ARGSTR_WATER, ARGSTR_CLOUD): # Read in mask raster, then unset bits that will not be used to mask. mask_select, mask_x, mask_y = rio.extractRasterData( maskFile, 'z', 'x', 'y') mask_ones = np.ones_like(mask_select) if not args.get(ARGSTR_EDGE): np.bitwise_and(mask_select, ~np.left_shift(mask_ones, MASKCOMP_EDGE_BIT), out=mask_select) if not args.get(ARGSTR_WATER): np.bitwise_and(mask_select, ~np.left_shift(mask_ones, MASKCOMP_WATER_BIT), out=mask_select) if not args.get(ARGSTR_CLOUD): np.bitwise_and(mask_select, ~np.left_shift(mask_ones, MASKCOMP_CLOUD_BIT), out=mask_select) del mask_ones # Convert remaining component bits to a binary boolean mask. mask_select = mask_select.astype(np.bool) else: mask_select = None # Make list for downsampled mask array "pyramids" # that may be built on-the-fly. mask_pyramids = [mask_select] mask_pyramid_current = mask_pyramids[0] # Apply mask to source raster images and save results. for src_suffix, maskval in suffix_maskval_dict.items(): bitmaskSuffix_temp = bitmaskSuffix if src_suffix.startswith(STRIP_LSF_PREFIX): if not bitmask_is_strip_lsf: continue elif bitmask_is_strip_lsf: bitmaskSuffix_temp = SUFFIX_STRIP_BITMASK src_rasterFile = maskFile.replace(bitmaskSuffix_temp, src_suffix) if '*' in src_rasterFile: src_rasterFile_list = glob.glob(src_rasterFile) if len(src_rasterFile_list) == 0: print("No source rasters found matching filename pattern: {}". format(src_rasterFile)) continue maskFile_globbed = None for src_rasterFile in src_rasterFile_list: if src_rasterFile.endswith(bitmaskSuffix_temp): maskFile_globbed = src_rasterFile break if maskFile_globbed is not None: src_rasterFile_list.remove(maskFile_globbed) else: src_rasterFile_list = [src_rasterFile] for src_rasterFile in src_rasterFile_list: if not os.path.isfile(src_rasterFile): print( "Source raster does not exist: {}".format(src_rasterFile)) continue dst_rasterFile = get_dstFile(src_rasterFile, args) if os.path.isfile(dst_rasterFile): print( "Output raster already exists: {}".format(dst_rasterFile)) if not args.get(ARGSTR_OVERWRITE): continue print("Masking source raster ({}) to output raster ({})".format( src_rasterFile, dst_rasterFile)) # Read in source raster. dst_array, src_nodataval = rio.extractRasterData( src_rasterFile, 'array', 'nodata_val') print("Source NoData value: {}".format(src_nodataval)) # Set masking value to source NoDataVal if necessary. if maskval is None: if src_nodataval is None: print( "Source raster does not have a set NoData value, " "so masking value cannot be automatically determined; skipping" ) continue else: maskval = src_nodataval print("Masking value: {}".format(maskval)) if mask_pyramid_current is not None: # Apply mask. if mask_pyramid_current.shape != dst_array.shape: # See if the required mask pyramid level has already been built. mask_pyramid_current = None for arr in mask_pyramids: if arr.shape == dst_array.shape: mask_pyramid_current = arr break if mask_pyramid_current is None: # Build the required mask pyramid level and add to pyramids. import lib.raster_array_tools as rat mask_pyramid_current = rat.imresize(mask_select, dst_array.shape, interp='nearest') mask_pyramids.append(mask_pyramid_current) dst_array[mask_pyramid_current] = maskval # Handle nodata options. if nodata_opt == ARGCHO_DST_NODATA_SAME: dst_nodataval = src_nodataval elif nodata_opt == ARGCHO_DST_NODATA_ADD: dst_nodataval = maskval if src_nodataval is None else src_nodataval elif nodata_opt == ARGCHO_DST_NODATA_SWITCH: dst_nodataval = maskval elif nodata_opt == ARGCHO_DST_NODATA_CONVERT: if src_nodataval is not None: dst_nodata = ((dst_array == src_nodataval) if not np.isnan(src_nodataval) else np.isnan(dst_array)) dst_array[dst_nodata] = maskval dst_nodataval = maskval elif nodata_opt == ARGCHO_DST_NODATA_UNSET: dst_nodataval = None print("Output NoData value: {}".format(dst_nodataval)) # Save output masked raster. rio.saveArrayAsTiff(dst_array, dst_rasterFile, nodata_val=dst_nodataval, like_raster=src_rasterFile, dtype_out=dst_array.dtype)