def make_buffered_point_raster_mask(shore_sample_point_vector_path, template_raster_path, workspace_dir, habitat_id, protective_distance, target_buffer_raster_path): gpkg_driver = ogr.GetDriverByName('GPKG') buffer_habitat_path = os.path.join(workspace_dir, '%s_buffer.gpkg' % habitat_id) buffer_habitat_vector = gpkg_driver.CreateDataSource(buffer_habitat_path) wgs84_srs = osr.SpatialReference() wgs84_srs.ImportFromEPSG(4326) buffer_habitat_layer = (buffer_habitat_vector.CreateLayer( habitat_id, wgs84_srs, ogr.wkbPolygon)) buffer_habitat_layer_defn = buffer_habitat_layer.GetLayerDefn() shore_sample_point_vector = gdal.OpenEx(shore_sample_point_vector_path, gdal.OF_VECTOR) shore_sample_point_layer = shore_sample_point_vector.GetLayer() buffer_habitat_layer.StartTransaction() for point_index, point_feature in enumerate(shore_sample_point_layer): if point_index % 1000 == 0: LOGGER.debug( 'point buffering is %.2f%% complete', point_index / shore_sample_point_layer.GetFeatureCount() * 100.0) # for each point, convert to local UTM to buffer out a given # distance then back to wgs84 point_geom = point_feature.GetGeometryRef() utm_srs = calculate_utm_srs(point_geom.GetX(), point_geom.GetY()) wgs84_to_utm_transform = osr.CoordinateTransformation( wgs84_srs, utm_srs) utm_to_wgs84_transform = osr.CoordinateTransformation( utm_srs, wgs84_srs) point_geom.Transform(wgs84_to_utm_transform) buffer_poly_geom = point_geom.Buffer(protective_distance) buffer_poly_geom.Transform(utm_to_wgs84_transform) buffer_point_feature = ogr.Feature(buffer_habitat_layer_defn) buffer_point_feature.SetGeometry(buffer_poly_geom) buffer_habitat_layer.CreateFeature(buffer_point_feature) buffer_point_feature = None point_feature = None buffer_poly_geom = None point_geom = None # at this point every shore point has been buffered to the effective # habitat distance and the habitat service has been saved with it buffer_habitat_layer.CommitTransaction() buffer_habitat_layer = None buffer_habitat_vector = None pygeoprocessing.new_raster_from_base( template_raster_path, target_buffer_raster_path, gdal.GDT_Float32, [0], raster_driver_creation_tuple=('GTIFF', ('TILED=YES', 'BIGTIFF=YES', 'COMPRESS=LZW', 'BLOCKXSIZE=256', 'BLOCKYSIZE=256', 'SPARSE_OK=TRUE'))) pygeoprocessing.rasterize(buffer_habitat_path, target_buffer_raster_path, burn_values=[1])
def rasterize_carbon_zones( base_raster_path, carbon_vector_path, rasterized_zones_path): """Rasterize carbon zones, expect 'CODE' as the parameter in the vector.""" pygeoprocessing.new_raster_from_base( base_raster_path, rasterized_zones_path, gdal.GDT_Int32, [-1]) pygeoprocessing.rasterize( carbon_vector_path, rasterized_zones_path, option_list=['ATTRIBUTE=CODE'])
def make_mask(base_raster_path, mask_vector_path, target_raster_path): """Mask vector onto target using base as the reference.""" pygeoprocessing.new_raster_from_base(base_raster_path, target_raster_path, gdal.GDT_Byte, [0], fill_value_list=[0]) pygeoprocessing.rasterize(mask_vector_path, target_raster_path, burn_values=[1])
def rasterize_streams( base_raster_path, stream_vector_path, target_streams_raster_path): """Rasterize streams.""" pygeoprocessing.new_raster_from_base( base_raster_path, target_streams_raster_path, gdal.GDT_Byte, [2], fill_value_list=[2]) LOGGER.debug(stream_vector_path) pygeoprocessing.rasterize( stream_vector_path, target_streams_raster_path, burn_values=[1])
def _clip_and_mask_dem(dem_path, aoi_path, target_path, working_dir): """Clip and mask the DEM to the AOI. Args: dem_path (string): The path to the DEM to use. Must have the same projection as the AOI. aoi_path (string): The path to the AOI to use. Must have the same projection as the DEM. target_path (string): The path on disk to where the clipped and masked raster will be saved. If a file exists at this location it will be overwritten. The raster will have a bounding box matching the intersection of the AOI and the DEM's bounding box and a spatial reference matching the AOI and the DEM. working_dir (string): A path to a directory on disk. A new temporary directory will be created within this directory for the storage of several working files. This temporary directory will be removed at the end of this function. Returns: ``None`` """ temp_dir = tempfile.mkdtemp(dir=working_dir, prefix='clip_dem') LOGGER.info('Clipping the DEM to its intersection with the AOI.') aoi_vector_info = pygeoprocessing.get_vector_info(aoi_path) dem_raster_info = pygeoprocessing.get_raster_info(dem_path) mean_pixel_size = ( abs(dem_raster_info['pixel_size'][0]) + abs(dem_raster_info['pixel_size'][1])) / 2.0 pixel_size = (mean_pixel_size, -mean_pixel_size) intersection_bbox = [op(aoi_dim, dem_dim) for (aoi_dim, dem_dim, op) in zip(aoi_vector_info['bounding_box'], dem_raster_info['bounding_box'], [max, max, min, min])] clipped_dem_path = os.path.join(temp_dir, 'clipped_dem.tif') pygeoprocessing.warp_raster( dem_path, pixel_size, clipped_dem_path, 'near', target_bb=intersection_bbox) LOGGER.info('Masking DEM pixels outside the AOI to nodata') aoi_mask_raster_path = os.path.join(temp_dir, 'aoi_mask.tif') pygeoprocessing.new_raster_from_base( clipped_dem_path, aoi_mask_raster_path, gdal.GDT_Byte, [_BYTE_NODATA], [0], raster_driver_creation_tuple=BYTE_GTIFF_CREATION_OPTIONS) pygeoprocessing.rasterize(aoi_path, aoi_mask_raster_path, [1], None) dem_nodata = dem_raster_info['nodata'][0] def _mask_op(dem, aoi_mask): valid_pixels = (~utils.array_equals_nodata(dem, dem_nodata) & (aoi_mask == 1)) masked_dem = numpy.empty(dem.shape) masked_dem[:] = dem_nodata masked_dem[valid_pixels] = dem[valid_pixels] return masked_dem pygeoprocessing.raster_calculator( [(clipped_dem_path, 1), (aoi_mask_raster_path, 1)], _mask_op, target_path, gdal.GDT_Float32, dem_nodata, raster_driver_creation_tuple=FLOAT_GTIFF_CREATION_OPTIONS) shutil.rmtree(temp_dir, ignore_errors=True)
def _collapse_infrastructure_layers(infrastructure_dir, base_raster_path, infrastructure_path, tmp_dir): """Collapse all GIS infrastructure layers to one raster. Gathers all the GIS layers in the given directory and collapses them to a single byte raster mask where 1 indicates a pixel overlapping with one of the original infrastructure layers, 0 does not, and nodata indicates a region that has no layers that overlap but are still contained in the bounding box. Parameters: infrastructure_dir (string): path to a directory containing maps of either gdal compatible rasters or OGR compatible shapefiles. base_raster_path (string): a path to a file that has the dimensions and projection of the desired output infrastructure file. infrastructure_path (string): (output) path to a file that will be a byte raster with 1s everywhere there was a GIS layer present in the GIS layers in `infrastructure_dir`. tmp_dir (string): path to folder to store inetermediate datasets such as aligned versions of infrastructure rasters. Returns: None """ # load the infrastructure layers from disk infrastructure_filenames = [] infrastructure_nodata_list = [] infrastructure_tmp_filenames = [] # in case we need to rasterize some vector inputs: tmp_rasterize_dir = os.path.join(tmp_dir, 'rasterized') for root_directory, _, filename_list in os.walk(infrastructure_dir): for filename in filename_list: if filename.lower().endswith(".tif"): infrastructure_filenames.append( os.path.join(root_directory, filename)) infrastructure_nodata_list.append( pygeoprocessing.get_raster_info( infrastructure_filenames[-1])['nodata'][0]) if filename.lower().endswith(".shp"): utils.make_directories([tmp_rasterize_dir]) file_handle, tmp_raster_path = tempfile.mkstemp( dir=tmp_rasterize_dir, suffix='.tif') os.close(file_handle) pygeoprocessing.new_raster_from_base(base_raster_path, tmp_raster_path, gdal.GDT_Int32, [-1.0], fill_value_list=[0]) pygeoprocessing.rasterize(os.path.join(root_directory, filename), tmp_raster_path, burn_values=[1], option_list=["ALL_TOUCHED=TRUE"]) infrastructure_filenames.append(tmp_raster_path) infrastructure_tmp_filenames.append(tmp_raster_path) infrastructure_nodata_list.append( pygeoprocessing.get_raster_info( infrastructure_filenames[-1])['nodata'][0]) if len(infrastructure_filenames) == 0: raise ValueError( "infrastructure directory didn't have any rasters or " "vectors at %s", infrastructure_dir) infrastructure_nodata = -1 def _collapse_infrastructure_op(*infrastructure_array_list): """For each pixel, create mask 1 if all valid, else set to nodata.""" nodata_mask = (numpy.isclose(infrastructure_array_list[0], infrastructure_nodata_list[0])) infrastructure_result = infrastructure_array_list[0] > 0 for index in range(1, len(infrastructure_array_list)): current_nodata = numpy.isclose(infrastructure_array_list[index], infrastructure_nodata_list[index]) infrastructure_result = (infrastructure_result | ( (infrastructure_array_list[index] > 0) & ~current_nodata)) nodata_mask = (nodata_mask & current_nodata) infrastructure_result[nodata_mask] = infrastructure_nodata return infrastructure_result LOGGER.info('collapse infrastructure into one raster') aligned_infrastructure_target_list = [ os.path.join(tmp_dir, os.path.basename(x)) for x in infrastructure_filenames ] base_raster_info = pygeoprocessing.get_raster_info(base_raster_path) pygeoprocessing.align_and_resize_raster_stack( infrastructure_filenames, aligned_infrastructure_target_list, ['near'] * len(infrastructure_filenames), base_raster_info['pixel_size'], base_raster_info['bounding_box']) infra_filename_band_list = [(x, 1) for x in aligned_infrastructure_target_list] pygeoprocessing.raster_calculator(infra_filename_band_list, _collapse_infrastructure_op, infrastructure_path, gdal.GDT_Byte, infrastructure_nodata) # clean up the temporary filenames if os.path.isdir(tmp_rasterize_dir): for filename in infrastructure_tmp_filenames: os.remove(filename) os.rmdir(tmp_rasterize_dir)
def execute(args): """Habitat Quality. Open files necessary for the portion of the habitat_quality model. Args: workspace_dir (string): a path to the directory that will write output and other temporary files (required) lulc_cur_path (string): a path to an input land use/land cover raster (required) lulc_fut_path (string): a path to an input land use/land cover raster (optional) lulc_bas_path (string): a path to an input land use/land cover raster (optional, but required for rarity calculations) threat_folder (string): a path to the directory that will contain all threat rasters (required) threats_table_path (string): a path to an input CSV containing data of all the considered threats. Each row is a degradation source and each column a different attribute of the source with the following names: 'THREAT','MAX_DIST','WEIGHT' (required). access_vector_path (string): a path to an input polygon shapefile containing data on the relative protection against threats (optional) sensitivity_table_path (string): a path to an input CSV file of LULC types, whether they are considered habitat, and their sensitivity to each threat (required) half_saturation_constant (float): a python float that determines the spread and central tendency of habitat quality scores (required) suffix (string): a python string that will be inserted into all raster path paths just before the file extension. Example Args Dictionary:: { 'workspace_dir': 'path/to/workspace_dir', 'lulc_cur_path': 'path/to/lulc_cur_raster', 'lulc_fut_path': 'path/to/lulc_fut_raster', 'lulc_bas_path': 'path/to/lulc_bas_raster', 'threat_raster_folder': 'path/to/threat_rasters/', 'threats_table_path': 'path/to/threats_csv', 'access_vector_path': 'path/to/access_shapefile', 'sensitivity_table_path': 'path/to/sensitivity_csv', 'half_saturation_constant': 0.5, 'suffix': '_results', } Returns: None """ workspace = args['workspace_dir'] # Append a _ to the suffix if it's not empty and doesn't already have one suffix = utils.make_suffix_string(args, 'suffix') # Check to see if each of the workspace folders exists. If not, create the # folder in the filesystem. inter_dir = os.path.join(workspace, 'intermediate') out_dir = os.path.join(workspace, 'output') kernel_dir = os.path.join(inter_dir, 'kernels') utils.make_directories([inter_dir, out_dir, kernel_dir]) # get a handle on the folder with the threat rasters threat_raster_dir = args['threat_raster_folder'] threat_dict = utils.build_lookup_from_csv(args['threats_table_path'], 'THREAT', to_lower=False) sensitivity_dict = utils.build_lookup_from_csv( args['sensitivity_table_path'], 'LULC', to_lower=False) # check that the required headers exist in the sensitivity table. # Raise exception if they don't. sens_header_list = sensitivity_dict.items()[0][1].keys() required_sens_header_list = ['LULC', 'NAME', 'HABITAT'] missing_sens_header_list = [ h for h in required_sens_header_list if h not in sens_header_list ] if missing_sens_header_list: raise ValueError('Column(s) %s are missing in the sensitivity table' % (', '.join(missing_sens_header_list))) # check that the threat names in the threats table match with the threats # columns in the sensitivity table. Raise exception if they don't. for threat in threat_dict: if 'L_' + threat not in sens_header_list: missing_threat_header_list = (set(sens_header_list) - set(required_sens_header_list)) raise ValueError( 'Threat "%s" does not match any column in the sensitivity ' 'table. Possible columns: %s' % (threat, missing_threat_header_list)) # get the half saturation constant try: half_saturation = float(args['half_saturation_constant']) except ValueError: raise ValueError('Half-saturation constant is not a numeric number.' 'It is: %s' % args['half_saturation_constant']) # declare dictionaries to store the land cover and the threat rasters # pertaining to the different threats lulc_path_dict = {} threat_path_dict = {} # also store land cover and threat rasters in a list lulc_and_threat_raster_list = [] aligned_raster_list = [] # declare a set to store unique codes from lulc rasters raster_unique_lucodes = set() # compile all the threat rasters associated with the land cover for lulc_key, lulc_args in (('_c', 'lulc_cur_path'), ('_f', 'lulc_fut_path'), ('_b', 'lulc_bas_path')): if lulc_args in args: lulc_path = args[lulc_args] lulc_path_dict[lulc_key] = lulc_path # save land cover paths in a list for alignment and resize lulc_and_threat_raster_list.append(lulc_path) aligned_raster_list.append( os.path.join( inter_dir, os.path.basename(lulc_path).replace( '.tif', '_aligned.tif'))) # save unique codes to check if it's missing in sensitivity table for _, lulc_block in pygeoprocessing.iterblocks((lulc_path, 1)): raster_unique_lucodes.update(numpy.unique(lulc_block)) # Remove the nodata value from the set of landuser codes. nodata = pygeoprocessing.get_raster_info(lulc_path)['nodata'][0] try: raster_unique_lucodes.remove(nodata) except KeyError: # KeyError when the nodata value was not encountered in the # raster's pixel values. Same result when nodata value is # None. pass # add a key to the threat dictionary that associates all threat # rasters with this land cover threat_path_dict['threat' + lulc_key] = {} # for each threat given in the CSV file try opening the associated # raster which should be found in threat_raster_folder for threat in threat_dict: # it's okay to have no threat raster for baseline scenario threat_path_dict['threat' + lulc_key][threat] = ( resolve_ambiguous_raster_path( os.path.join(threat_raster_dir, threat + lulc_key), raise_error=(lulc_key != '_b'))) # save threat paths in a list for alignment and resize threat_path = threat_path_dict['threat' + lulc_key][threat] if threat_path: lulc_and_threat_raster_list.append(threat_path) aligned_raster_list.append( os.path.join( inter_dir, os.path.basename(lulc_path).replace( '.tif', '_aligned.tif'))) # check if there's any lucode from the LULC rasters missing in the # sensitivity table table_unique_lucodes = set(sensitivity_dict.keys()) missing_lucodes = raster_unique_lucodes.difference(table_unique_lucodes) if missing_lucodes: raise ValueError( 'The following land cover codes were found in your landcover rasters ' 'but not in your sensitivity table. Check your sensitivity table ' 'to see if they are missing: %s. \n\n' % ', '.join([str(x) for x in sorted(missing_lucodes)])) # Align and resize all the land cover and threat rasters, # and tore them in the intermediate folder LOGGER.info('Starting aligning and resizing land cover and threat rasters') lulc_pixel_size = (pygeoprocessing.get_raster_info( args['lulc_cur_path']))['pixel_size'] aligned_raster_list = [ os.path.join(inter_dir, os.path.basename(path).replace('.tif', '_aligned.tif')) for path in lulc_and_threat_raster_list ] pygeoprocessing.align_and_resize_raster_stack( lulc_and_threat_raster_list, aligned_raster_list, ['near'] * len(lulc_and_threat_raster_list), lulc_pixel_size, 'intersection') LOGGER.info('Finished aligning and resizing land cover and threat rasters') # Modify paths in lulc_path_dict and threat_path_dict to be aligned rasters for lulc_key, lulc_path in lulc_path_dict.iteritems(): lulc_path_dict[lulc_key] = os.path.join( inter_dir, os.path.basename(lulc_path).replace('.tif', '_aligned.tif')) for threat in threat_dict: threat_path = threat_path_dict['threat' + lulc_key][threat] if threat_path in lulc_and_threat_raster_list: threat_path_dict['threat' + lulc_key][threat] = os.path.join( inter_dir, os.path.basename(threat_path).replace( '.tif', '_aligned.tif')) LOGGER.info('Starting habitat_quality biophysical calculations') # Rasterize access vector, if value is null set to 1 (fully accessible), # else set to the value according to the ACCESS attribute cur_lulc_path = lulc_path_dict['_c'] fill_value = 1.0 try: LOGGER.info('Handling Access Shape') access_raster_path = os.path.join(inter_dir, 'access_layer%s.tif' % suffix) # create a new raster based on the raster info of current land cover pygeoprocessing.new_raster_from_base(cur_lulc_path, access_raster_path, gdal.GDT_Float32, [_OUT_NODATA], fill_value_list=[fill_value]) pygeoprocessing.rasterize(args['access_vector_path'], access_raster_path, burn_values=None, option_list=['ATTRIBUTE=ACCESS']) except KeyError: LOGGER.info('No Access Shape Provided, access raster filled with 1s.') # calculate the weight sum which is the sum of all the threats' weights weight_sum = 0.0 for threat_data in threat_dict.itervalues(): # Sum weight of threats weight_sum = weight_sum + threat_data['WEIGHT'] LOGGER.debug('lulc_path_dict : %s', lulc_path_dict) # for each land cover raster provided compute habitat quality for lulc_key, lulc_path in lulc_path_dict.iteritems(): LOGGER.info('Calculating habitat quality for landuse: %s', lulc_path) # Create raster of habitat based on habitat field habitat_raster_path = os.path.join( inter_dir, 'habitat%s%s.tif' % (lulc_key, suffix)) map_raster_to_dict_values(lulc_path, habitat_raster_path, sensitivity_dict, 'HABITAT', _OUT_NODATA, values_required=False) # initialize a list that will store all the threat/threat rasters # after they have been adjusted for distance, weight, and access deg_raster_list = [] # a list to keep track of the normalized weight for each threat weight_list = numpy.array([]) # variable to indicate whether we should break out of calculations # for a land cover because a threat raster was not found exit_landcover = False # adjust each threat/threat raster for distance, weight, and access for threat, threat_data in threat_dict.iteritems(): LOGGER.info('Calculating threat: %s.\nThreat data: %s' % (threat, threat_data)) # get the threat raster for the specific threat threat_raster_path = threat_path_dict['threat' + lulc_key][threat] LOGGER.info('threat_raster_path %s', threat_raster_path) if threat_raster_path is None: LOGGER.info( 'The threat raster for %s could not be found for the land ' 'cover %s. Skipping Habitat Quality calculation for this ' 'land cover.' % (threat, lulc_key)) exit_landcover = True break # need the pixel size for the threat raster so we can create # an appropriate kernel for convolution threat_pixel_size = pygeoprocessing.get_raster_info( threat_raster_path)['pixel_size'] # pixel size tuple could have negative value mean_threat_pixel_size = (abs(threat_pixel_size[0]) + abs(threat_pixel_size[1])) / 2.0 # convert max distance (given in KM) to meters max_dist_m = threat_data['MAX_DIST'] * 1000.0 # convert max distance from meters to the number of pixels that # represents on the raster max_dist_pixel = max_dist_m / mean_threat_pixel_size LOGGER.debug('Max distance in pixels: %f', max_dist_pixel) # blur the threat raster based on the effect of the threat over # distance decay_type = threat_data['DECAY'] kernel_path = os.path.join( kernel_dir, 'kernel_%s%s%s.tif' % (threat, lulc_key, suffix)) if decay_type == 'linear': make_linear_decay_kernel_path(max_dist_pixel, kernel_path) elif decay_type == 'exponential': utils.exponential_decay_kernel_raster(max_dist_pixel, kernel_path) else: raise ValueError( "Unknown type of decay in biophysical table, should be " "either 'linear' or 'exponential'. Input was %s for threat" " %s." % (decay_type, threat)) filtered_threat_raster_path = os.path.join( inter_dir, 'filtered_%s%s%s.tif' % (threat, lulc_key, suffix)) pygeoprocessing.convolve_2d((threat_raster_path, 1), (kernel_path, 1), filtered_threat_raster_path) # create sensitivity raster based on threat sens_raster_path = os.path.join( inter_dir, 'sens_%s%s%s.tif' % (threat, lulc_key, suffix)) map_raster_to_dict_values(lulc_path, sens_raster_path, sensitivity_dict, 'L_' + threat, _OUT_NODATA, values_required=True) # get the normalized weight for each threat weight_avg = threat_data['WEIGHT'] / weight_sum # add the threat raster adjusted by distance and the raster # representing sensitivity to the list to be past to # vectorized_rasters below deg_raster_list.append(filtered_threat_raster_path) deg_raster_list.append(sens_raster_path) # store the normalized weight for each threat in a list that # will be used below in total_degradation weight_list = numpy.append(weight_list, weight_avg) # check to see if we got here because a threat raster was missing # and if so then we want to skip to the next landcover if exit_landcover: continue def total_degradation(*raster): """A vectorized function that computes the degradation value for each pixel based on each threat and then sums them together *rasters - a list of floats depicting the adjusted threat value per pixel based on distance and sensitivity. The values are in pairs so that the values for each threat can be tracked: [filtered_val_threat1, sens_val_threat1, filtered_val_threat2, sens_val_threat2, ...] There is an optional last value in the list which is the access_raster value, but it is only present if access_raster is not None. returns - the total degradation score for the pixel""" # we can not be certain how many threats the user will enter, # so we handle each filtered threat and sensitivity raster # in pairs sum_degradation = numpy.zeros(raster[0].shape) for index in range(len(raster) / 2): step = index * 2 sum_degradation += (raster[step] * raster[step + 1] * weight_list[index]) nodata_mask = numpy.empty(raster[0].shape, dtype=numpy.int8) nodata_mask[:] = 0 for array in raster: nodata_mask = nodata_mask | (array == _OUT_NODATA) # the last element in raster is access return numpy.where(nodata_mask, _OUT_NODATA, sum_degradation * raster[-1]) # add the access_raster onto the end of the collected raster list. The # access_raster will be values from the shapefile if provided or a # raster filled with all 1's if not deg_raster_list.append(access_raster_path) deg_sum_raster_path = os.path.join( out_dir, 'deg_sum' + lulc_key + suffix + '.tif') LOGGER.info('Starting raster calculation on total_degradation') deg_raster_band_list = [(path, 1) for path in deg_raster_list] pygeoprocessing.raster_calculator(deg_raster_band_list, total_degradation, deg_sum_raster_path, gdal.GDT_Float32, _OUT_NODATA) LOGGER.info('Finished raster calculation on total_degradation') # Compute habitat quality # ksq: a term used below to compute habitat quality ksq = half_saturation**_SCALING_PARAM def quality_op(degradation, habitat): """Vectorized function that computes habitat quality given a degradation and habitat value. degradation - a float from the created degradation raster above. habitat - a float indicating habitat suitability from from the habitat raster created above. returns - a float representing the habitat quality score for a pixel """ degredataion_clamped = numpy.where(degradation < 0, 0, degradation) return numpy.where( (degradation == _OUT_NODATA) | (habitat == _OUT_NODATA), _OUT_NODATA, (habitat * (1.0 - ((degredataion_clamped**_SCALING_PARAM) / (degredataion_clamped**_SCALING_PARAM + ksq))))) quality_path = os.path.join(out_dir, 'quality' + lulc_key + suffix + '.tif') LOGGER.info('Starting raster calculation on quality_op') deg_hab_raster_list = [deg_sum_raster_path, habitat_raster_path] deg_hab_raster_band_list = [(path, 1) for path in deg_hab_raster_list] pygeoprocessing.raster_calculator(deg_hab_raster_band_list, quality_op, quality_path, gdal.GDT_Float32, _OUT_NODATA) LOGGER.info('Finished raster calculation on quality_op') # Compute Rarity if user supplied baseline raster if '_b' not in lulc_path_dict: LOGGER.info('Baseline not provided to compute Rarity') else: lulc_base_path = lulc_path_dict['_b'] # get the area of a base pixel to use for computing rarity where the # pixel sizes are different between base and cur/fut rasters base_pixel_size = pygeoprocessing.get_raster_info( lulc_base_path)['pixel_size'] base_area = float(abs(base_pixel_size[0]) * abs(base_pixel_size[1])) base_nodata = pygeoprocessing.get_raster_info( lulc_base_path)['nodata'][0] lulc_code_count_b = raster_pixel_count(lulc_base_path) # compute rarity for current landscape and future (if provided) for lulc_key in ['_c', '_f']: if lulc_key not in lulc_path_dict: continue lulc_path = lulc_path_dict[lulc_key] lulc_time = 'current' if lulc_key == '_c' else 'future' # get the area of a cur/fut pixel lulc_pixel_size = pygeoprocessing.get_raster_info( lulc_path)['pixel_size'] lulc_area = float( abs(lulc_pixel_size[0]) * abs(lulc_pixel_size[1])) lulc_nodata = pygeoprocessing.get_raster_info( lulc_path)['nodata'][0] def trim_op(base, cover_x): """Trim cover_x to the mask of base. Parameters: base (numpy.ndarray): base raster from 'lulc_base' cover_x (numpy.ndarray): either future or current land cover raster from 'lulc_path' above Returns: _OUT_NODATA where either array has nodata, otherwise cover_x. """ return numpy.where( (base == base_nodata) | (cover_x == lulc_nodata), base_nodata, cover_x) LOGGER.info('Create new cover for %s', lulc_path) new_cover_path = os.path.join( inter_dir, 'new_cover' + lulc_key + suffix + '.tif') LOGGER.info('Starting masking %s land cover to base land cover.' % lulc_time) pygeoprocessing.raster_calculator([(lulc_base_path, 1), (lulc_path, 1)], trim_op, new_cover_path, gdal.GDT_Float32, _OUT_NODATA) LOGGER.info('Finished masking %s land cover to base land cover.' % lulc_time) LOGGER.info('Starting rarity computation on %s land cover.' % lulc_time) lulc_code_count_x = raster_pixel_count(new_cover_path) # a dictionary to map LULC types to a number that depicts how # rare they are considered code_index = {} # compute rarity index for each lulc code # define 0.0 if an lulc code is found in the cur/fut landcover # but not the baseline for code in lulc_code_count_x.iterkeys(): if code in lulc_code_count_b: numerator = lulc_code_count_x[code] * lulc_area denominator = lulc_code_count_b[code] * base_area ratio = 1.0 - (numerator / denominator) code_index[code] = ratio else: code_index[code] = 0.0 rarity_path = os.path.join(out_dir, 'rarity' + lulc_key + suffix + '.tif') pygeoprocessing.reclassify_raster((new_cover_path, 1), code_index, rarity_path, gdal.GDT_Float32, _RARITY_NODATA) LOGGER.info('Finished rarity computation on %s land cover.' % lulc_time) LOGGER.info('Finished habitat_quality biophysical calculations')
def calculate_reef_value(shore_sample_point_vector, template_raster_path, reef_habitat_raster_path, working_dir, target_reef_value_raster_path): """Calculate habitat value. Will create rasters in the `working_dir` directory named from the `habitat_fieldname_list` values containing relative importance of global habitat. The higher the value of a pixel the more important that pixel of habitat is for protection of the coastline. Parameters: shore_sample_point_vector (str): path to CV analysis vector containing at least the fields `Rt` and `Rt_nohab_[hab]` for all habitat types under consideration. template_raster_path (str): path to an existing raster whose size and shape will be used to be the base of the raster that's created for each habitat type. habitat_fieldname_list (list): list of habitat ids to analyise. habitat_vector_path_map (dict): maps fieldnames from `habitat_fieldname_list` to 3-tuples of (path to hab vector (str), risk val (float), protective distance (float)). working_dir (str): path to directory containing habitat back projection results target_reef_value_raster_path (str): path to raster value raster for that habitat. Returns: None. """ temp_workspace_dir = os.path.join(working_dir, 'hab_value_churn') for dir_path in [working_dir, temp_workspace_dir]: try: os.makedirs(dir_path) except OSError: pass gpkg_driver = ogr.GetDriverByName('gpkg') shore_sample_point_vector = gdal.OpenEx(shore_sample_point_vector, gdal.OF_VECTOR) shore_sample_point_layer = shore_sample_point_vector.GetLayer() reef_service_id = 'Rt_habservice_reefs_all' buffer_habitat_path = os.path.join(temp_workspace_dir, 'reefs_all_buffer.gpkg') buffer_habitat_vector = gpkg_driver.CreateDataSource(buffer_habitat_path) wgs84_srs = osr.SpatialReference() wgs84_srs.ImportFromEPSG(4326) buffer_habitat_layer = (buffer_habitat_vector.CreateLayer( reef_service_id, wgs84_srs, ogr.wkbPolygon)) buffer_habitat_layer.CreateField( ogr.FieldDefn(reef_service_id, ogr.OFTReal)) buffer_habitat_layer_defn = buffer_habitat_layer.GetLayerDefn() shore_sample_point_layer.ResetReading() buffer_habitat_layer.StartTransaction() for point_index, point_feature in enumerate(shore_sample_point_layer): if point_index % 1000 == 0: LOGGER.debug( 'point buffering is %.2f%% complete', point_index / shore_sample_point_layer.GetFeatureCount() * 100.0) # for each point, convert to local UTM to buffer out a given # distance then back to wgs84 point_geom = point_feature.GetGeometryRef() if point_geom.GetX() > 178 or point_geom.GetX() < -178: continue utm_srs = calculate_utm_srs(point_geom.GetX(), point_geom.GetY()) wgs84_to_utm_transform = osr.CoordinateTransformation( wgs84_srs, utm_srs) utm_to_wgs84_transform = osr.CoordinateTransformation( utm_srs, wgs84_srs) point_geom.Transform(wgs84_to_utm_transform) buffer_poly_geom = point_geom.Buffer(REEF_PROT_DIST) buffer_poly_geom.Transform(utm_to_wgs84_transform) buffer_point_feature = ogr.Feature(buffer_habitat_layer_defn) buffer_point_feature.SetGeometry(buffer_poly_geom) buffer_point_feature.SetField( reef_service_id, point_feature.GetField('Rt_habservice_reefs_all')) buffer_habitat_layer.CreateFeature(buffer_point_feature) buffer_point_feature = None point_feature = None buffer_poly_geom = None point_geom = None # at this point every shore point has been buffered to the effective # habitat distance and the habitat service has been saved with it buffer_habitat_layer.CommitTransaction() buffer_habitat_layer = None buffer_habitat_vector = None value_coverage_raster_path = os.path.join(temp_workspace_dir, 'reefs_all_value_cover.tif') pygeoprocessing.new_raster_from_base( template_raster_path, value_coverage_raster_path, gdal.GDT_Float32, [0], raster_driver_creation_tuple=('GTIFF', ('TILED=YES', 'BIGTIFF=YES', 'COMPRESS=LZW', 'BLOCKXSIZE=256', 'BLOCKYSIZE=256', 'SPARSE_OK=TRUE'))) pygeoprocessing.rasterize( buffer_habitat_path, value_coverage_raster_path, option_list=['ATTRIBUTE=%s' % reef_service_id, 'MERGE_ALG=ADD']) value_coverage_nodata = pygeoprocessing.get_raster_info( value_coverage_raster_path)['nodata'][0] hab_nodata = pygeoprocessing.get_raster_info( reef_habitat_raster_path)['nodata'][0] wgs84_srs = osr.SpatialReference() wgs84_srs.ImportFromEPSG(4326) aligned_value_hab_raster_path_list = align_raster_list( [value_coverage_raster_path, reef_habitat_raster_path], temp_workspace_dir, target_sr_wkt=wgs84_srs.ExportToWkt()) pygeoprocessing.raster_calculator( [(aligned_value_hab_raster_path_list[0], 1), (aligned_value_hab_raster_path_list[1], 1), (value_coverage_nodata, 'raw'), (hab_nodata, 'raw')], intersect_raster_op, target_reef_value_raster_path, gdal.GDT_Float32, value_coverage_nodata) ecoshard.build_overviews(target_reef_value_raster_path)
def alternative_index_workflow(workspace_dir, raster_input_dict, aoi_path, index_path, polygon_input_list=None): """Compute the alternative index from raw inputs. All inputs, including AOI, must be share coordinate reference system and must have roughly equivalent extents. Recommend that inputs are clipped and projected in Arc prior to running this script. Args: workspace_dir (string): path to workspace where intermediate results should be created/stored raster_input_dict (dict): a nested python dictionary containing info about raster-based inputs that should be combined. The keys in the index should be the labels for each input; values in the dictionary should be dictionaries containing the keys 'path' (path to the raster input) and 'weight' (weighting value that is applied to the normalized values in this input relative to others). EACH INDEX IS INTERPRETED AS HIGH VALUE = GOOD. aoi_path (string): path to boundary of the study area index_path (string): path to location where the index should be saved polygon_input_list (list): list of paths to polygon inputs that should be included. Each of these is assigned a weight of 1. Side effects: creates or modifies a raster at the location ``index_path`` Returns: None """ # ensure that each new input shares spatial reference vector_info = pygeoprocessing.get_vector_info(aoi_path) destination_proj = osr.SpatialReference() destination_proj.ImportFromWkt(vector_info['projection_wkt']) problem_list = [] for new_input in raster_input_dict: new_proj = osr.SpatialReference() new_proj.ImportFromWkt( pygeoprocessing.get_raster_info( raster_input_dict[new_input]['path'])['projection_wkt']) if (new_proj.IsSame(destination_proj) == 0): problem_list.append(new_input) if problem_list: raise ValueError( "Project these to match the AOI: {}".format(problem_list)) intermediate_dir = os.path.join(workspace_dir, 'intermediate') if not os.path.exists(intermediate_dir): os.makedirs(intermediate_dir) normalized_dir = os.path.join(intermediate_dir, 'normalized') if not os.path.exists(normalized_dir): os.makedirs(normalized_dir) aligned_dir = os.path.join(intermediate_dir, 'aligned') if not os.path.exists(aligned_dir): os.makedirs(aligned_dir) # normalize all raster-based inputs within AOI base_raster_path_list = [] aligned_raster_path_list = [] for new_input in raster_input_dict: value_raster_path = raster_input_dict[new_input]['path'] try: weight = raster_input_dict[new_input]['weight'] except KeyError: weight = 1 bn = os.path.basename(value_raster_path) normalized_path = os.path.join(normalized_dir, bn) aligned_path = os.path.join(aligned_dir, bn) base_raster_path_list.append(normalized_path) aligned_raster_path_list.append(aligned_path) if not os.path.exists(normalized_path): with tempfile.NamedTemporaryFile( prefix='mask_raster', delete=False, suffix='.tif', dir=normalized_dir) as clipped_raster_file: clipped_raster_path = clipped_raster_file.name pygeoprocessing.mask_raster((value_raster_path, 1), aoi_path, clipped_raster_path) normalize(clipped_raster_path, normalized_path, aoi_path, weight) os.remove(clipped_raster_path) # align and resample normalized rasters, using minimum pixel size of inputs pixel_size_list = [] for new_input in raster_input_dict: value_raster_path = raster_input_dict[new_input]['path'] raster_info = pygeoprocessing.get_raster_info(value_raster_path) pixel_size_list.append(raster_info['pixel_size']) target_pixel_size = min(pixel_size_list) min_pixel_index = pixel_size_list.index(min(pixel_size_list)) if not all([os.path.exists(f) for f in aligned_raster_path_list]): pygeoprocessing.align_and_resize_raster_stack( base_raster_path_list, aligned_raster_path_list, ['near'] * len(base_raster_path_list), target_pixel_size, 'intersection', raster_align_index=min_pixel_index) # rasterize polygon inputs template_raster_path = aligned_raster_path_list[0] if polygon_input_list: for vec_path in polygon_input_list: target_raster_path = os.path.join( aligned_dir, '{}.tif'.format(os.path.basename(vec_path)[:-4])) aligned_raster_path_list.append(target_raster_path) if not os.path.exists(target_raster_path): pygeoprocessing.new_raster_from_base( template_raster_path, target_raster_path, gdal.GDT_Int16, [_TARGET_NODATA], fill_value_list=[_TARGET_NODATA]) pygeoprocessing.rasterize(vec_path, target_raster_path, burn_values=[100]) # add together raster_list_sum(aligned_raster_path_list, _TARGET_NODATA, index_path, _TARGET_NODATA, nodata_remove=True)
def normalize_by_polygon(raster_path, vector_path, percentile, clamp_range, workspace_dir, target_path): """Normalize a raster locally by regions defined by vector. Parameters: raster_path (str): path to base raster to aggregate over. vector_path (str): path to a vector that defines local regions to normalize over. Any pixels outside of these polygons will be set to nodata. percentile (float): a number in the range [0, 100] that is used to normalize the local regions defined by `vector_path`. This number will be used to calculate the percentile in each region separately clamp_range (list or tuple): a min/max range to clamp the normalized result by. workspace_dir (str): path to a workspace to create and keep intermediate files. Returns: None. """ base_dir = os.path.dirname(target_path) for dir_path in [base_dir, workspace_dir]: try: os.makedirs(dir_path) except OSError: pass vector = ogr.Open(vector_path) layer = vector.GetLayer() fid_to_percentile_pickle_path = {} for feature in layer: # clip the original layer and then mask it fid = feature.GetFID() feature_mask_path = os.path.join(workspace_dir, '%d_mask.tif' % fid) mask_raster_task = TASK_GRAPH.add_task( func=clip_and_mask_raster, args=(raster_path, vector_path, fid, feature_mask_path), target_path_list=[feature_mask_path], task_name='mask feature %d' % fid) percentile_pickle_path = os.path.join( workspace_dir, '%d_%d.pickle' % (fid, percentile)) _ = TASK_GRAPH.add_task(func=calculate_percentile, args=(feature_mask_path, [percentile], base_dir, percentile_pickle_path), target_path_list=[percentile_pickle_path], dependent_task_list=[mask_raster_task], task_name='calculating %s' % percentile_pickle_path) fid_to_percentile_pickle_path[fid] = percentile_pickle_path feature = None local_vector_path = os.path.join(workspace_dir, 'local_vector.gpkg') gpkg_driver = ogr.GetDriverByName('GPKG') local_vector = gpkg_driver.CopyDataSource(vector, local_vector_path) vector = None layer = None local_layer = local_vector.GetLayer() local_layer.CreateField(ogr.FieldDefn('norm_val', ogr.OFTReal)) global_norm_value_raster_path = os.path.join(workspace_dir, 'global_norm_values.tif') pygeoprocessing.new_raster_from_base( raster_path, global_norm_value_raster_path, gdal.GDT_Float32, [-1], raster_driver_creation_tuple=('GTIFF', ('TILED=YES', 'BIGTIFF=YES', 'COMPRESS=ZSTD', 'BLOCKXSIZE=256', 'BLOCKYSIZE=256', 'SPARSE_OK=TRUE'))) TASK_GRAPH.join() for fid, pickle_path in fid_to_percentile_pickle_path.items(): feature = local_layer.GetFeature(fid) with open(pickle_path, 'rb') as pickle_file: percentile_list = pickle.load(pickle_file) if len(percentile_list) > 0: feature.SetField('norm_val', percentile_list[0]) else: feature.SetField('norm_val', -1.0) local_layer.SetFeature(feature) feature = None local_layer = None local_vector = None pygeoprocessing.rasterize(local_vector_path, global_norm_value_raster_path, option_list=['ATTRIBUTE=norm_val']) raster_nodata = pygeoprocessing.get_raster_info(raster_path)['nodata'][0] pygeoprocessing.raster_calculator([(raster_path, 1), (global_norm_value_raster_path, 1), (clamp_range, 'raw'), (raster_nodata, 'raw'), (-1, 'raw'), (-1, 'raw')], divide_op, target_path, gdal.GDT_Float32, -1)
def _mask_raster_by_vector( base_raster_path_band, vector_path, working_dir, target_raster_path): """Mask pixels outside of the vector to nodata. Parameters: base_raster_path (string): path/band tuple to raster to process vector_path (string): path to single layer raster that is used to indicate areas to preserve from the base raster. Areas outside of this vector are set to nodata. working_dir (str): path to temporary directory. target_raster_path (string): path to a single band raster that will be created of the same dimensions and data type as `base_raster_path_band` where any pixels that lie outside of `vector_path` coverage will be set to nodata. Returns: None. """ # Warp input raster to be same bounding box as AOI if smaller. base_raster_info = pygeoprocessing.get_raster_info( base_raster_path_band[0]) nodata = base_raster_info['nodata'][base_raster_path_band[1]-1] target_pixel_size = base_raster_info['pixel_size'] vector_info = pygeoprocessing.get_vector_info(vector_path) target_bounding_box = pygeoprocessing.merge_bounding_box_list( [base_raster_info['bounding_box'], vector_info['bounding_box']], 'intersection') pygeoprocessing.warp_raster( base_raster_path_band[0], target_pixel_size, target_raster_path, 'near', target_bb=target_bounding_box) # Create mask raster same size as the warped raster. tmp_dir = tempfile.mkdtemp(dir=working_dir) mask_raster_path = os.path.join(tmp_dir, 'mask.tif') pygeoprocessing.new_raster_from_base( target_raster_path, mask_raster_path, gdal.GDT_Byte, [0], fill_value_list=[0]) # Rasterize the vector onto the mask raster pygeoprocessing.rasterize(vector_path, mask_raster_path, [1], None) # Parallel iterate over warped raster and mask raster to mask out original. target_raster = gdal.OpenEx( target_raster_path, gdal.GA_Update | gdal.OF_RASTER) target_band = target_raster.GetRasterBand(1) mask_raster = gdal.OpenEx(mask_raster_path, gdal.OF_RASTER) mask_band = mask_raster.GetRasterBand(1) for offset_dict in pygeoprocessing.iterblocks( (mask_raster_path, 1), offset_only=True): data_array = target_band.ReadAsArray(**offset_dict) mask_array = mask_band.ReadAsArray(**offset_dict) data_array[mask_array != 1] = nodata target_band.WriteArray( data_array, xoff=offset_dict['xoff'], yoff=offset_dict['yoff']) target_band.FlushCache() target_band = None target_raster = None mask_band = None mask_raster = None try: shutil.rmtree(tmp_dir) except OSError: LOGGER.warn("Unable to delete temporary file %s", mask_raster_path)
if len(set([habitat_vector_epsg, shoreline_epsg, aoi_epsg])) > 1: raise ValueError( "AOI raster, shoreline point vector, and habitat vector do not " "all share the same projection") # Rasterize CV points w/ value using target AOI as mask pre_mask_point_raster_path = os.path.join(args.workspace_dir, 'pre_mask_shore_points.tif') target_pixel_size = aoi_raster_info['pixel_size'] pygeoprocessing.new_raster_from_base(args.aoi_mask_raster_path, pre_mask_point_raster_path, gdal.GDT_Float32, [numpy.finfo(numpy.float32).min]) pygeoprocessing.rasterize( args.shoreline_point_vector_path, pre_mask_point_raster_path, option_list=[ f'ATTRIBUTE={args.shoreline_vector_fieldname}', 'ALL_TOUCHED=TRUE' ]) # TODO: mask out values that are not in a defined AOI. shore_point_raster_path = os.path.join( args.workspace_dir, args.target_shoreline_raster_filename) mask_by_nodata(pre_mask_point_raster_path, args.aoi_mask_raster_path, shore_point_raster_path) # Create habitat mask habitat_mask_raster_path = os.path.join(args.workspace_dir, 'habitat_mask.tif') pygeoprocessing.new_raster_from_base(args.aoi_mask_raster_path, habitat_mask_raster_path, gdal.GDT_Byte, [0])
def _calculate_ipcc_biomass( landcover_raster_path, churn_dir, target_biomass_raster_path): """Calculate IPCC method for biomass for given landcover. Args: landcover_raster_path (str): path to ESA landcover raster. churn_dir (str): path to use for temporary files. target_biomass_raster_path (str): path to raster to create target biomass (not in density) Return: None """ def _ipcc_carbon_op( lulc_array, zones_array, zone_lulc_to_carbon_map): """Map carbon to LULC/zone values and multiply by conversion map.""" result = numpy.zeros(lulc_array.shape) for zone_id in numpy.unique(zones_array): if zone_id in zone_lulc_to_carbon_map: zone_mask = zones_array == zone_id result[zone_mask] = ( zone_lulc_to_carbon_map[zone_id][lulc_array[zone_mask]]) return result def _parse_carbon_lulc_table(ipcc_carbon_table_path): """Parse out the IPCC carbon table by zone and lulc.""" with open(IPCC_CARBON_TABLE_PATH, 'r') as carbon_table_file: header_line = carbon_table_file.readline() lulc_code_list = [ int(lucode) for lucode in header_line.split(',')[1:]] max_code = max(lulc_code_list) zone_lucode_to_carbon_map = {} for line in carbon_table_file: split_line = line.split(',') if split_line[0] == '': continue zone_id = int(split_line[0]) zone_lucode_to_carbon_map[zone_id] = numpy.zeros(max_code+1) for lucode, carbon_value in zip( lulc_code_list, split_line[1:]): zone_lucode_to_carbon_map[zone_id][lucode] = float( carbon_value) return zone_lucode_to_carbon_map rasterized_zones_raster_path = os.path.join(churn_dir, 'carbon_zones.tif') LOGGER.info( f'rasterize carbon zones of {landcover_raster_path} to ' f'{rasterized_zones_raster_path}') pygeoprocessing.new_raster_from_base( landcover_raster_path, rasterized_zones_raster_path, gdal.GDT_Int32, [-1]) pygeoprocessing.rasterize( CARBON_ZONES_VECTOR_PATH, rasterized_zones_raster_path, option_list=['ATTRIBUTE=CODE']) zone_lucode_to_carbon_map = _parse_carbon_lulc_table( IPCC_CARBON_TABLE_PATH) biomass_per_ha_raster_path = os.path.join(churn_dir, 'biomass_per_ha.tif') pygeoprocessing.raster_calculator( [(landcover_raster_path, 1), (rasterized_zones_raster_path, 1), (zone_lucode_to_carbon_map, 'raw')], _ipcc_carbon_op, biomass_per_ha_raster_path, gdal.GDT_Float32, -1) density_per_ha_to_total_per_pixel( biomass_per_ha_raster_path, 1.0, target_biomass_raster_path)