def main(gis_ws, cdl_year='', block_size=16384, mask_flag=False, overwrite_flag=False, pyramids_flag=False, stats_flag=False): """Mask DEM values for non-agricultural pixels Use CDL derived agmask (in CDL workspace) to define agricultural pixels Args: gis_ws (str): Folder/workspace path of the GIS data for the project cdl_year (str): Comma separated list and/or range of years block_size (int): Maximum block size to use for raster processing mask_flag (bool): If True, mask pixels outside extent shapefile overwrite_flag (bool): If True, overwrite existing files pyramids_flag (bool): If True, build pyramids/overviews for the output rasters stats_flag (bool): If True, compute statistics for the output rasters Returns: None """ logging.info('\nExtracting Agriculatural DEM Values') input_dem_name = 'ned_30m_albers.img' cdl_format = '{}_30m_cdls.img' dem_ws = os.path.join(gis_ws, 'dem') cdl_ws = os.path.join(gis_ws, 'cdl') scratch_ws = os.path.join(gis_ws, 'scratch') zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img') output_format = 'HFA' if pyramids_flag: levels = '2 4 8 16 32 64 128' # gdal.SetConfigOption('USE_RRD', 'YES') # gdal.SetConfigOption('HFA_USE_RRD', 'YES') # gdal.SetConfigOption('HFA_COMPRESS_OVR', 'YES') if os.name == 'posix': shell_flag = False else: shell_flag = True # Check input folders if not os.path.isdir(gis_ws): logging.error('\nERROR: The GIS workspace does not exist' '\n {}'.format(gis_ws)) sys.exit() elif not os.path.isdir(cdl_ws): logging.error('\nERROR: The CDL workspace does not exist' '\n {}'.format(cdl_ws)) sys.exit() elif not os.path.isdir(dem_ws): logging.error('\nERROR: The DEM workspace does not exist' '\n {}'.format(dem_ws)) sys.exit() elif mask_flag and not os.path.isfile(zone_raster_path): logging.error( '\nERROR: The zone raster {} does not exist\n' ' Try re-running "clip_cdl_raster.py"'.format(zone_raster_path)) sys.exit() logging.info('\nGIS Workspace: {}'.format(gis_ws)) logging.info('CDL Workspace: {}'.format(cdl_ws)) logging.info('DEM Workspace: {}\n'.format(dem_ws)) # Check input files input_dem_path = os.path.join(dem_ws, input_dem_name) if not os.path.isfile(input_dem_path): logging.error( '\nERROR: The raster {} does not exist'.format(input_dem_path)) sys.exit() # Process existing dem rasters (from merge_dems.py) input_rows, input_cols = gdc.raster_path_shape(input_dem_path) # Process each CDL year separately for cdl_year in list(util.parse_int_set(cdl_year)): logging.info('\n{}'.format(cdl_year)) # cdl_path = os.path.join(cdl_ws, cdl_format.format(cdl_year)) output_dem_path = os.path.join(dem_ws, 'dem_{}_30m_cdls.img'.format(cdl_year)) # agland_path = os.path.join( # cdl_ws, 'agland_{}_30m_cdls.img'.format(cdl_year)) agmask_path = os.path.join(cdl_ws, 'agmask_{}_30m_cdls.img'.format(cdl_year)) if not os.path.isfile(agmask_path): logging.error('\nERROR: The ag-mask raster {} does not exist\n' ' Try re-running "build_ag_cdl_rasters.py"'.format( agmask_path)) continue # Copy input DEM if overwrite_flag and os.path.isfile(output_dem_path): subprocess.check_output( ['gdalmanage', 'delete', '-f', output_format, output_dem_path], shell=shell_flag) if (os.path.isfile(input_dem_path) and not os.path.isfile(output_dem_path)): logging.info('\nCopying DEM raster') logging.debug('{}'.format(input_dem_path)) subprocess.check_output([ 'gdal_translate', '-of', output_format, '-co', 'COMPRESSED=YES', input_dem_path, output_dem_path ], shell=shell_flag) # Set non-ag areas to nodata value logging.debug('Processing by block') logging.debug(' Input cols/rows: {0}/{1}'.format( input_cols, input_rows)) for b_i, b_j in gdc.block_gen(input_rows, input_cols, block_size): logging.debug(' Block y: {0:5d} x: {1:5d}'.format(b_i, b_j)) # Read in data for block agmask_array = gdc.raster_to_block(agmask_path, b_i, b_j, block_size, return_nodata=False) dem_array, dem_nodata = gdc.raster_to_block(input_dem_path, b_i, b_j, block_size, return_nodata=True) # Mask CDL values outside extent shapefile if mask_flag and os.path.isfile(zone_raster_path): zone_array = gdc.raster_to_block(zone_raster_path, b_i, b_j, block_size) dem_array[zone_array == 0] = dem_nodata # Set dem values for non-ag pixels to nodata dem_array[~agmask_array.astype(np.bool)] = dem_nodata gdc.block_to_raster(dem_array, output_dem_path, b_i, b_j, block_size) del agmask_array, dem_array, dem_nodata if stats_flag and os.path.isfile(output_dem_path): logging.info('Computing statistics') logging.debug(' {}'.format(output_dem_path)) subprocess.check_output( ['gdalinfo', '-stats', '-nomd', output_dem_path], shell=shell_flag) if pyramids_flag and os.path.isfile(output_dem_path): logging.info('Building pyramids') logging.debug(' {}'.format(output_dem_path)) subprocess.check_output(['gdaladdo', '-ro', output_dem_path] + levels.split(), shell=shell_flag)
def main(gis_ws, input_soil_ws, cdl_year='', prop_list=['all'], block_size=16384, mask_flag=False, overwrite_flag=False, pyramids_flag=False, stats_flag=False): """Mask soil values for non-agricultural pixels Use CDL derived agmask (in CDL workspace) to define agricultural pixels Args: gis_ws (str): Folder/workspace path of the GIS data for the project input_soil_ws (str): Folder/workspace path of the common soils data cdl_year (str): Comma separated list and/or range of years prop_list (list): String of the soil types to build (i.e. awc, clay, sand, all) block_size (int): Maximum block size to use for raster processing mask_flag (bool): If True, mask pixels outside extent shapefile overwrite_flag (bool): If True, overwrite existing files pyramids_flag (bool): If True, build pyramids/overviews for the output rasters stats_flag (bool): If True, compute statistics for the output rasters Returns: None """ logging.info('\nExtracting Agriculatural Soil Values') input_soil_fmt = '{}_30m_albers.img' # cdl_format = '{0}_30m_cdls.img' cdl_ws = os.path.join(gis_ws, 'cdl') # input_soil_ws = os.path.join(gis_ws, 'statsgo') output_soil_ws = os.path.join(gis_ws, 'soils') scratch_ws = os.path.join(gis_ws, 'scratch') zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img') output_format = 'HFA' output_nodata = -9999 if pyramids_flag: levels = '2 4 8 16 32 64 128' # gdal.SetConfigOption('USE_RRD', 'YES') # gdal.SetConfigOption('HFA_USE_RRD', 'YES') # gdal.SetConfigOption('HFA_COMPRESS_OVR', 'YES') if os.name == 'posix': shell_flag = False else: shell_flag = True # Check input folders if not os.path.isdir(gis_ws): logging.error('\nERROR: The GIS folder does not exist' '\n {}'.format(gis_ws)) sys.exit() elif not os.path.isdir(cdl_ws): logging.error('\nERROR: The CDL folder does not exist' '\n {}'.format(cdl_ws)) sys.exit() elif not os.path.isdir(input_soil_ws): logging.error('\nERROR: The input soil folder does not exist' '\n {}'.format(input_soil_ws)) sys.exit() elif mask_flag and not os.path.isfile(zone_raster_path): logging.error( '\nERROR: The zone raster {} does not exist\n' ' Try re-running "clip_cdl_raster.py"'.format(zone_raster_path)) sys.exit() logging.info('\nGIS Workspace: {}'.format(gis_ws)) logging.info('CDL Workspace: {}'.format(cdl_ws)) logging.info('Input Soil Workspace: {}'.format(input_soil_ws)) logging.info('Output Soil Workspace: {}'.format(output_soil_ws)) # Process each CDL year separately for cdl_year in list(util.parse_int_set(cdl_year)): logging.info('\n{}'.format(cdl_year)) # cdl_path = os.path.join(cdl_ws, cdl_format.format(cdl_year)) output_soil_fmt = '{}_{}_30m_cdls.img'.format('{}', cdl_year) # agland_path = os.path.join( # cdl_ws, 'agland_{}_30m_cdls.img'.format(cdl_year)) agmask_path = os.path.join(cdl_ws, 'agmask_{}_30m_cdls.img'.format(cdl_year)) if not os.path.isfile(agmask_path): logging.error('\nERROR: The ag-mask raster {} does not exist\n' + ' Try re-running "build_ag_cdl_rasters.py"'.format( agmask_path)) continue logging.info('Soil Property: {}'.format(', '.join(prop_list))) if prop_list == ['all']: prop_list = ['AWC', 'CLAY', 'SAND'] # Process existing soil rasters (from rasterize script) for prop_str in prop_list: logging.info('\nSoil: {}'.format(prop_str.upper())) input_soil_path = os.path.join(input_soil_ws, input_soil_fmt.format(prop_str)) output_soil_path = os.path.join( output_soil_ws, output_soil_fmt.format(prop_str.lower())) if not os.path.isfile(input_soil_path): logging.error('\nERROR: The raster {} does not exist'.format( input_soil_path)) continue # Create a copy of the input raster to modify if overwrite_flag and os.path.isfile(output_soil_path): subprocess.check_output([ 'gdalmanage', 'delete', '-f', output_format, output_soil_path ], shell=shell_flag) if (os.path.isfile(input_soil_path) and not os.path.isfile(output_soil_path)): logging.info('\nCopying soil raster') logging.debug('{}'.format(input_soil_path)) subprocess.check_output([ 'gdal_translate', '-of', output_format, '-co', 'COMPRESSED=YES', input_soil_path, output_soil_path ], shell=shell_flag) # Get the size of the input raster input_rows, input_cols = gdc.raster_path_shape(input_soil_path) # Set non-ag areas to nodata value logging.debug('Processing by block') logging.debug(' Input cols/rows: {0}/{1}'.format( input_cols, input_rows)) for b_i, b_j in gdc.block_gen(input_rows, input_cols, block_size): logging.debug(' Block y: {0:5d} x: {1:5d}'.format(b_i, b_j)) # Read in data for block agmask_array = gdc.raster_to_block(agmask_path, b_i, b_j, block_size, return_nodata=False) soil_array, soil_nodata = gdc.raster_to_block( input_soil_path, b_i, b_j, block_size, return_nodata=True) # Mask CDL values outside extent shapefile if mask_flag and os.path.isfile(zone_raster_path): zone_array = gdc.raster_to_block(zone_raster_path, b_i, b_j, block_size) soil_array[zone_array == 0] = output_nodata # Set soil values for non-ag pixels to nodata soil_array[~agmask_array.astype(np.bool)] = output_nodata gdc.block_to_raster(soil_array, output_soil_path, b_i, b_j, block_size) del agmask_array, soil_array, soil_nodata # Override the output raster nodata value output_ds = gdal.Open(output_soil_path, 1) output_band = output_ds.GetRasterBand(1) output_band.SetNoDataValue(output_nodata) output_ds = None if stats_flag and os.path.isfile(output_soil_path): logging.info('Computing statistics') logging.debug(' {}'.format(output_soil_path)) subprocess.check_output( ['gdalinfo', '-stats', '-nomd', output_soil_path], shell=shell_flag) if pyramids_flag and os.path.isfile(output_soil_path): logging.info('Building pyramids') logging.debug(' {}'.format(output_soil_path)) subprocess.check_output(['gdaladdo', '-ro', output_soil_path] + levels.split(), shell=shell_flag)
def main(ini_path, overwrite_flag=False): """Build CDL shapefiles for agricultural pixels Parameters ---------- ini_path : str File path of the parameter INI file. overwrite_flag : bool If True, overwrite existing shapefile (the default is False). Returns ------- None """ logging.info('\nBuilding Agricultural CDL Shapefile') logging.debug('INI: {}'.format(ini_path)) config = util.read_ini(ini_path, section='CROP_ET') zone_path = config.get('CROP_ET', 'cells_path') crop_path = config.get('CROP_ET', 'crop_path') temp_path = crop_path.replace('.shp', '_temp.shp') cdl_ws = config.get('CROP_ET', 'cdl_folder') cdl_year = int(config.get('CROP_ET', 'cdl_year')) cdl_format = config.get('CROP_ET', 'cdl_format') # It might make more sense to pass the non-ag CDL values instead cdl_crops = util.parse_int_set(config.get('CROP_ET', 'cdl_crops')) # cdl_nonag = util.parse_int_set(config.get('CROP_ET', 'cdl_nonag')) cdl_path = os.path.join(cdl_ws, cdl_format.format(cdl_year, 'img')) # Output field name in the crops shapefile crop_field = config.get('CROP_ET', 'crop_field') shp_driver = ogr.GetDriverByName('ESRI Shapefile') if os.path.isfile(crop_path): if overwrite_flag: shp_driver.DeleteDataSource(crop_path) else: return True if not os.path.isfile(zone_path): logging.error( '\nERROR: The ET zone shapefile doesn\'t exist, exiting\n' ' {}'.format(zone_path)) sys.exit() elif not os.path.isfile(cdl_path): logging.error('\nERROR: The CDL raster doesn\'t exist, exiting\n' ' {}'.format(cdl_path)) sys.exit() logging.debug('Zones: {}'.format(zone_path)) # CDL Raster Properties cdl_ds = gdal.Open(cdl_path) cdl_band = cdl_ds.GetRasterBand(1) try: cdl_nodata = int(cdl_band.GetNoDataValue()) except TypeError: cdl_nodata = 0 cdl_gtype = cdl_band.DataType cdl_proj = cdl_ds.GetProjection() cdl_osr = gdc.proj_osr(cdl_proj) cdl_geo = cdl_ds.GetGeoTransform() cdl_x, cdl_y = gdc.geo_origin(cdl_geo) cdl_cs = gdc.geo_cellsize(cdl_geo, x_only=True) cdl_extent = gdc.raster_ds_extent(cdl_ds) logging.debug('\nCDL Raster Properties') logging.debug(' Geo: {}'.format(cdl_geo)) logging.debug(' Snap: {} {}'.format(cdl_x, cdl_y)) logging.debug(' Cellsize: {}'.format(cdl_cs)) logging.debug(' Nodata: {}'.format(cdl_nodata)) logging.debug(' GDAL Type: {}'.format(cdl_gtype)) logging.debug(' Extent: {}'.format(cdl_extent)) logging.debug(' Projection: {}'.format(cdl_osr.ExportToWkt())) # logging.debug(' OSR: {}'.format(cdl_osr)) # ET Zones Properties zone_ds = shp_driver.Open(zone_path, 0) zone_lyr = zone_ds.GetLayer() zone_osr = zone_lyr.GetSpatialRef() zone_wkt = gdc.osr_proj(zone_osr) zone_extent = gdc.feature_lyr_extent(zone_lyr) logging.debug('\nET Zones Shapefile Properties') logging.debug(' Extent: {}'.format(zone_extent)) logging.debug(' Projection: {}'.format(zone_osr.ExportToWkt())) # logging.debug(' OSR: {}'.format(zones_osr)) if zone_osr.IsGeographic(): logging.error('\nERROR: The ET zones shapefile must be in a projected ' 'coordinate system, exiting') sys.exit() # Subset/clip properties # Project the extent to the CDL spatial reference logging.debug('\nClip Subset') clip_extent = zone_extent.project(zone_osr, cdl_osr) logging.debug(' Projected: {}'.format(clip_extent)) # Adjust the clip extent to the CDL snap point and cell size clip_extent.buffer(10 * cdl_cs) clip_extent.adjust_to_snap(snap_x=cdl_x, snap_y=cdl_y, cs=cdl_cs, method='EXPAND') logging.debug(' Snapped: {}'.format(clip_extent)) # Limit the subset extent to the CDL extent clip_extent.clip(cdl_extent) logging.debug(' Clipped: {}'.format(clip_extent)) # Compute the clip geotransform and shape clip_geo = clip_extent.geo(cs=cdl_cs) clip_rows, clip_cols = clip_extent.shape(cs=cdl_cs) logging.debug(' Rows/Cols: {} {}'.format(clip_rows, clip_cols)) # Building a raster mask was a little more efficient than selecting # touching features later on. logging.debug('\nBuilding ET Zones mask') zone_count = zone_lyr.GetFeatureCount() if zone_count < 255: zone_mask_gtype = gdal.GDT_Byte zone_mask_nodata = 255 elif zone_count < 65535: zone_mask_gtype = gdal.GDT_UInt16 zone_mask_nodata = 65535 else: zone_mask_gtype = gdal.GDT_UInt32 zone_mask_nodata = 4294967295 memory_driver = gdal.GetDriverByName('GTiff') # zones_mask_ds = memory_driver.Create( # os.path.join(os.path.dirname(zones_path), 'zones_mask.tiff'), # clip_cols, clip_rows, 1, zones_mask_gtype) memory_driver = gdal.GetDriverByName('MEM') zone_mask_ds = memory_driver.Create('', clip_cols, clip_rows, 1, zone_mask_gtype) zone_mask_ds.SetProjection(cdl_proj) zone_mask_ds.SetGeoTransform(clip_geo) zone_mask_band = zone_mask_ds.GetRasterBand(1) zone_mask_band.Fill(zone_mask_nodata) zone_mask_band.SetNoDataValue(zone_mask_nodata) gdal.RasterizeLayer(zone_mask_ds, [1], zone_lyr, burn_values=[1]) # zones_mask_ds = None # zones_mask_band = zones_mask_ds.GetRasterBand(1) zone_mask = zone_mask_band.ReadAsArray(0, 0, clip_cols, clip_rows) zone_mask = (zone_mask != zone_mask_nodata) zone_mask_ds = None logging.debug('\nBuilding initial CDL polygon shapefile') if os.path.isfile(temp_path): shp_driver.DeleteDataSource(temp_path) polygon_ds = shp_driver.CreateDataSource(temp_path) polygon_lyr = polygon_ds.CreateLayer('OUTPUT_POLY', geom_type=ogr.wkbPolygon) field_defn = ogr.FieldDefn(crop_field, ogr.OFTInteger) polygon_lyr.CreateField(field_defn) # TODO: Process CDL by tile # logging.debug('\nProcessing CDL by tile') # tile_list = [[0, 0]] # for tile_i, tile_j in tile_list: # logging.debug(' Tile: {} {}'.format(tile_i, tile_j)) logging.debug('\nConverting CDL raster to polygon') # Read the CDL subset array clip_xi, clip_yi = array_geo_offsets(cdl_geo, clip_geo) logging.debug(' Subset i/j: {} {}'.format(clip_xi, clip_yi)) cdl_array = cdl_band.ReadAsArray(clip_xi, clip_yi, clip_cols, clip_rows) cdl_ds = None # Apply the zones mask if np.any(zone_mask): cdl_array[~zone_mask] = cdl_nodata # Set non-agricultural pixels to nodata logging.debug('\nMasking non-crop pixels') cdl_array_values = np.unique(cdl_array) nodata_mask = np.zeros(cdl_array.shape, dtype=np.bool) for value in range(1, 255): if value in cdl_crops: continue elif value not in cdl_array_values: continue # logging.debug(' Value: {}'.format(value)) nodata_mask |= (cdl_array == value) cdl_array[nodata_mask] = cdl_nodata # # DEADBEEF - This is using the remap ranges # # It is probably more efficient than processing each crop separately # nodata_mask = np.zeros(cdl_array.shape, dtype=np.bool) # for [start, end, value] in cdl_agmask_remap: # if value == 1: # continue # logging.debug([start, end, value]) # nodata_mask |= (cdl_array >= start) & (cdl_array <= end) # cdl_array[nodata_mask] = cdl_nodata # Create an in-memory raster to read the CDL into # Set the mask band separately memory_driver = gdal.GetDriverByName('MEM') memory_ds = memory_driver.Create('', clip_cols, clip_rows, 2, cdl_gtype) memory_ds.SetGeoTransform(clip_geo) memory_ds.SetProjection(cdl_proj) memory_band = memory_ds.GetRasterBand(1) memory_band.SetNoDataValue(cdl_nodata) mask_band = memory_ds.GetRasterBand(2) # Write the CDL subset array to the memory raster logging.debug('\nWriting array') memory_band.WriteArray(cdl_array, 0, 0) mask_band.WriteArray(cdl_array != cdl_nodata, 0, 0) # Polygonize the CDL array logging.debug('\nConverting raster to polygon') gdal.Polygonize(memory_band, mask_band, polygon_lyr, 0) # Cleanup mask_band = None memory_band = None memory_ds = None polygon_lyr = None polygon_ds = None del cdl_array, nodata_mask, zone_mask # Write projection/spatial reference prj_osr = gdc.proj_osr(cdl_proj) prj_osr.MorphToESRI() with open(temp_path.replace('.shp', '.prj'), 'w') as prj_f: prj_f.write(prj_osr.ExportToWkt()) # Project crops to zones spatial reference logging.debug('\nProjecting crops to ET zones spatial reference') # ogr2ogr.project(temp_path, crop_path, zones_wkt) arcpy.project(temp_path, crop_path, zone_osr) logging.debug('\nRemoving temporary crops shapefile') arcpy.delete(temp_path)
def main(ini_path, area_threshold=10, dairy_cuttings=5, beef_cuttings=4, crop_str='', overwrite_flag=False): """Build a feature class for each crop and set default crop parameters Apply the values in the CropParams.txt as defaults to every cell Parameters ---------- ini_path : str File path of the parameter INI file. area_threshold : float CDL area threshold [acres]. dairy_cuttings : int Initial number of dairy hay cuttings. beef_cuttings : int Initial number of beef hay cuttings. crop_str : str Comma separated list or range of crops to compare (no spaces, ex: 1,2,4-6) overwrite_flag : bool If True, overwrite existing output rasters. Returns ------- None """ logging.info('\nCalculating ET-Demands Spatial Crop Parameters') remove_empty_flag = True # Input paths # DEADBEEF - For now, get cropET folder from INI file # This function may eventually be moved into the main cropET code crop_et_sec = 'CROP_ET' config = util.read_ini(ini_path, section=crop_et_sec) try: project_ws = config.get(crop_et_sec, 'project_folder') except: logging.error('project_folder parameter must be set in the INI file, ' 'exiting') return False try: gis_ws = config.get(crop_et_sec, 'gis_folder') except: logging.error('gis_folder parameter must be set in the INI file, ' 'exiting') return False try: cells_path = config.get(crop_et_sec, 'cells_path') except: # cells_path = os.path.join(gis_ws, 'ETCells.shp') logging.error('et_cells_path parameter must be set in the INI file, ' 'exiting') return False try: stations_path = config.get(crop_et_sec, 'stations_path') except: logging.error('stations_path parameter must be set in the INI file, ' 'exiting') return False try: crop_params_name = config.get(crop_et_sec, 'crop_params_name') except: logging.error( 'crop_params_name parameter must be set in the INI file, ' 'exiting') return False crop_et_ws = config.get(crop_et_sec, 'crop_et_folder') bin_ws = os.path.join(crop_et_ws, 'bin') try: calibration_ws = config.get(crop_et_sec, 'spatial_cal_folder') except: calibration_ws = os.path.join(project_ws, 'calibration') # Sub folder names static_ws = os.path.join(project_ws, 'static') crop_params_path = os.path.join(static_ws, crop_params_name) # ET cells field names cell_id_field = 'CELL_ID' cell_name_field = 'CELL_NAME' crop_acres_field = 'CROP_ACRES' # Only keep the following ET Cell fields keep_field_list = [cell_id_field, cell_name_field, 'AG_ACRES'] # keep_field_list = ['CELL_ID', 'STATION_ID', 'HUC8', 'HUC10', 'GRIDMET_ID', # 'COUNTYNAME', 'AG_ACRES'] # keep_field_list = ['FIPS', 'COUNTYNAME'] # Check input folders if not os.path.isdir(crop_et_ws): logging.error('\nERROR: The INI cropET folder does not exist' '\n {}'.format(crop_et_ws)) sys.exit() elif not os.path.isdir(bin_ws): logging.error('\nERROR: The bin workspace does not exist' '\n {}'.format(bin_ws)) sys.exit() elif not os.path.isdir(project_ws): logging.error('\nERROR: The project folder does not exist' '\n {}'.format(project_ws)) sys.exit() elif not os.path.isdir(gis_ws): logging.error('\nERROR: The GIS folder does not exist' '\n {}'.format(gis_ws)) sys.exit() if '.gdb' not in calibration_ws and not os.path.isdir(calibration_ws): os.makedirs(calibration_ws) logging.info('\nGIS Workspace: {}'.format(gis_ws)) logging.info('Project Workspace: {}'.format(project_ws)) logging.info('CropET Workspace: {}'.format(crop_et_ws)) logging.info('Bin Workspace: {}'.format(bin_ws)) logging.info('Calib. Workspace: {}'.format(calibration_ws)) # Check input files if not os.path.isfile(crop_params_path): logging.error('\nERROR: The crop parameters file does not exist' '\n {}'.format(crop_params_path)) sys.exit() elif not os.path.isfile(cells_path): logging.error('\nERROR: The ET Cell shapefile does not exist' '\n {}'.format(cells_path)) sys.exit() elif not os.path.isfile(stations_path): logging.error('\nERROR: The weather station shapefile does not exist' '\n {}'.format(stations_path)) sys.exit() logging.debug('Crop Params Path: {}'.format(crop_params_path)) logging.debug('ET Cells Path: {}'.format(cells_path)) logging.debug('Stations Path: {}'.format(stations_path)) # For now, only allow calibration parameters in separate shapefiles ext = '.shp' # # Build output geodatabase if necessary # if calibration_ws.endswith('.gdb'): # logging.debug('GDB Path: {}'.format(calibration_ws)) # ext = '' # _arcpy.exists(calibration_ws) and overwrite_flag: # try: _arcpy.delete(calibration_ws) # except: pass # if calibration_ws is not None and not _arcpy.exists(calibration_ws): # arcpy.CreateFileGDB_management( # os.path.dirname(calibration_ws), # os.path.basename(calibration_ws)) # else: # ext = '.shp' # Field Name, Property, Field Type # Property is the string of the CropParameter class property value # It will be used to access the property using getattr dairy_cutting_field = 'Dairy_Cut' beef_cutting_field = 'Beef_Cut' param_list = [ # ['Name', 'name', ogr.OFTString], # ['ClassNum', 'class_number', ogr.OFTInteger], # ['IsAnnual', 'is_annual', 'SHORT'], # ['IrrigFlag', 'irrigation_flag', 'SHORT'], # ['IrrigDays', 'days_after_planting_irrigation', ogr.OFTInteger], # ['Crop_FW', 'crop_fw', ogr.OFTInteger], # ['WinterCov', 'winter_surface_cover_class', 'SHORT'], # ['CropKcMax', 'kc_max', ogr.OFTReal], ['MAD_Init', 'mad_initial', ogr.OFTInteger], ['MAD_Mid', 'mad_midseason', ogr.OFTInteger], # ['RootDepIni', 'rooting_depth_initial', ogr.OFTReal], # ['RootDepMax', 'rooting_depth_max', ogr.OFTReal], # ['EndRootGrw', 'end_of_root_growth_fraction_time', ogr.OFTReal], # ['HeightInit', 'height_initial', ogr.OFTReal], # ['HeightMax', 'height_max', ogr.OFTReal], # ['CurveNum', 'curve_number', ogr.OFTInteger], # ['CurveName', 'curve_name', ogr.OFTString], # ['CurveType', 'curve_type', 'SHORT'], # ['PL_GU_Flag', 'flag_for_means_to_estimate_pl_or_gu', 'SHORT'], ['T30_CGDD', 't30_for_pl_or_gu_or_cgdd', ogr.OFTReal], ['PL_GU_Date', 'date_of_pl_or_gu', ogr.OFTReal], ['CGDD_Tbase', 'tbase', ogr.OFTReal], ['CGDD_EFC', 'cgdd_for_efc', ogr.OFTInteger], ['CGDD_Term', 'cgdd_for_termination', ogr.OFTInteger], ['Time_EFC', 'time_for_efc', ogr.OFTInteger], ['Time_Harv', 'time_for_harvest', ogr.OFTInteger], ['KillFrostC', 'killing_frost_temperature', ogr.OFTReal], # ['InvokeStrs', 'invoke_stress', 'SHORT'], # ['CN_Coarse', 'cn_coarse_soil', ogr.OFTInteger], # ['CN_Medium', 'cn_medium_soil', ogr.OFTInteger], # ['CN_Fine', 'cn_fine_soil', ogr.OFTInteger] ] # if calibration_ws.endswith('.gdb'): # dairy_cutting_field = 'Dairy_Cuttings' # beef_cutting_field = 'Beef_Cuttings' # param_list = [ # # ['Name', 'name', 'STRING'], # # ['Class_Number', 'class_number', ogr.OFTInteger], # # ['Is_Annual', 'is_annual', 'SHORT'], # # ['Irrigation_Flag', 'irrigation_flag', 'SHORT'], # # ['Irrigation_Days', 'days_after_planting_irrigation', ogr.OFTInteger], # # ['Crop_FW', 'crop_fw', ogr.OFTInteger], # # ['Winter_Cover_Class', 'winter_surface_cover_class', 'SHORT'], # # ['Crop_Kc_Max', 'kc_max', ogr.OFTReal], # # ['MAD_Initial', 'mad_initial', ogr.OFTInteger], # # ['MAD_Midseason', 'mad_midseason', ogr.OFTInteger], # # ['Root_Depth_Ini', 'rooting_depth_initial', ogr.OFTReal], # # ['Root_Depth_Max', 'rooting_depth_max', ogr.OFTReal], # # ['End_Root_Growth', 'end_of_root_growth_fraction_time', ogr.OFTReal], # # ['Height_Initial', 'height_initial', ogr.OFTReal], # # ['Height_Maximum', 'height_max', ogr.OFTReal], # # ['Curve_Number', 'curve_number', ogr.OFTInteger], # # ['Curve_Name', 'curve_name', ogr.OFTString], # # ['Curve_Type', 'curve_type', 'SHORT'], # # ['PL_GU_Flag', 'flag_for_means_to_estimate_pl_or_gu', 'SHORT'], # ['T30_CGDD', 't30_for_pl_or_gu_or_cgdd', ogr.OFTReal], # ['PL_GU_Date', 'date_of_pl_or_gu', ogr.OFTReal], # ['CGDD_Tbase', 'tbase', ogr.OFTReal], # ['CGDD_EFC', 'cgdd_for_efc', ogr.OFTInteger], # ['CGDD_Termination', 'cgdd_for_termination', ogr.OFTInteger], # ['Time_EFC', 'time_for_efc', ogr.OFTInteger], # ['Time_Harvest', 'time_for_harvest', ogr.OFTInteger], # ['Killing_Crost_C', 'killing_frost_temperature', ogr.OFTReal], # # ['Invoke_Stress', 'invoke_stress', 'SHORT'], # # ['CN_Coarse_Soil', 'cn_coarse_soil', ogr.OFTInteger], # # ['CN_Medium_Soil', 'cn_medium_soil', ogr.OFTInteger], # # ['CN_Fine_Soil', 'cn_fine_soil', ogr.OFTInteger] # ] crop_add_list = [] if crop_str: try: crop_add_list = sorted(list(util.parse_int_set(crop_str))) # try: # crop_test_list = sorted(list(set( # crop_test_list + list(util.parse_int_set(crop_str))) except: pass # Don't build crop parameter files for non-crops crop_skip_list = sorted(list(set([44, 45, 46, 55, 56, 57]))) # crop_test_list = sorted(list(set(crop_test_list + [46]))) logging.info('\ncrop_add_list = {}'.format(crop_add_list)) # Read crop parameters using ET Demands functions/methods logging.info('\nReading default crop parameters') sys.path.append(bin_ws) import crop_parameters crop_param_dict = crop_parameters.read_crop_parameters(crop_params_path) # Get list of crops specified in ET cells # Currently this may only be crops with CDL acreage crop_field_list = sorted([ field for field in _arcpy.list_fields(cells_path) if re.match('CROP_\d{2}', field) ]) crop_number_list = [int(f.split('_')[-1]) for f in crop_field_list] logging.info('Cell crop numbers: {}'.format(', '.join( list(util.ranges(crop_number_list))))) logging.debug('Cell crop fields: {}'.format(', '.join(crop_field_list))) # Get crop acreages for each cell # DEADBEEF - Does this dict need to be keyed by crop then cell_id? # Could it be changed to cell_id, crop or fid, crop to make it easier to # write to the shapefile using update_cursor()? crop_acreage_dict = defaultdict(dict) field_list = [cell_id_field] + crop_field_list for fid, row in _arcpy.search_cursor(cells_path, field_list).items(): for crop_field, crop_num in zip(crop_field_list, crop_number_list): if crop_skip_list and crop_num in crop_skip_list: continue elif crop_num in crop_add_list: crop_acreage_dict[crop_num][row[cell_id_field]] = 0 elif row[crop_field]: crop_acreage_dict[crop_num][ row[cell_id_field]] = row[crop_field] else: crop_acreage_dict[crop_num][row[cell_id_field]] = 0 crop_number_list = sorted(list(set(crop_number_list) | set(crop_add_list))) # Make an empty template crop feature class logging.info('') crop_template_path = os.path.join(calibration_ws, 'crop_00_template' + ext) if overwrite_flag and _arcpy.exists(crop_template_path): logging.debug('Overwriting template crop feature class') _arcpy.delete(crop_template_path) if _arcpy.exists(crop_template_path): logging.info('Template crop feature class already exists, skipping') else: logging.info('Building template crop feature class') _arcpy.copy(cells_path, crop_template_path) # Remove unneeded et cell fields for field in _arcpy.list_fields(crop_template_path): # if (field not in keep_field_list and # field.editable and not field.required): if field not in keep_field_list: logging.debug(' Delete field: {}'.format(field)) _arcpy.delete_field(crop_template_path, field) field_list = _arcpy.list_fields(crop_template_path) # Add crop acreage field if crop_acres_field not in field_list: logging.debug(' Add field: {}'.format(crop_acres_field)) _arcpy.add_field(crop_template_path, crop_acres_field, ogr.OFTReal) _arcpy.calculate_field(crop_template_path, crop_acres_field, '0') # Add crop parameter fields if necessary for param_field, param_method, param_type in param_list: logging.debug(' Add field: {}'.format(param_field)) if param_field not in field_list: _arcpy.add_field(crop_template_path, param_field, param_type) # if dairy_cutting_field not in field_list: # logging.debug(' Add field: {}'.format(dairy_cutting_field)) # _arcpy.add_field(crop_template_path, dairy_cutting_field, # ogr.OFTInteger) # _arcpy.calculate_field(crop_template_path, dairy_cutting_field, # dairy_cuttings) # if beef_cutting_field not in field_list: # logging.debug(' Add field: {}'.format(beef_cutting_field)) # _arcpy.add_field(crop_template_path, beef_cutting_field, # ogr.OFTInteger) # _arcpy.calculate_field(crop_template_path, beef_cutting_field, # beef_cuttings) # Add an empty/zero crop field for the field mappings below # if 'CROP_EMPTY' not in _arcpy.list_fields(cells_path): # _arcpy.add_field(cells_path, 'CROP_EMPTY', ogr.OFTReal) # _arcpy.calculate_field(cells_path, 'CROP_EMPTY', '0') # Process each crop logging.info('\nBuilding crop feature classes') for crop_num in crop_number_list: try: crop_param = crop_param_dict[crop_num] except: continue logging.info('{:>2d} {}'.format(crop_num, crop_param.name)) logging.debug('{}'.format(crop_param)) # Replace other characters with spaces, then remove multiple spaces crop_name = re.sub('[-"().,/~]', ' ', str(crop_param.name).lower()) crop_name = ' '.join(crop_name.strip().split()).replace(' ', '_') crop_path = os.path.join( calibration_ws, 'crop_{0:02d}_{1}{2}'.format(crop_num, crop_name, ext)) # crop_field = 'CROP_{:02d}'.format(crop_num) # Don't check crops in add list if crop_num in crop_add_list: pass # Skip if all zone crop areas are below threshold elif all( [v < area_threshold for v in crop_acreage_dict[crop_num].values()]): logging.info('** Skipping Crop {}, All crop acreages below' ' threshold'.format(crop_num)) continue # Remove existing shapefiles if necessary if overwrite_flag and _arcpy.exists(crop_path): logging.debug(' Overwriting: {}'.format( os.path.basename(crop_path))) _arcpy.delete(crop_path) # Don't check skip list until after existing files are removed # if ((crop_test_list and crop_num not in crop_test_list) or # _skip_list and crop_num in crop_skip_list)): # .debug(' Skipping') # Copy ET cells for each crop if needed if _arcpy.exists(crop_path): logging.debug(' Shapefile already exists, skipping') continue else: # logging.debug(' {}'.format(crop_path)) _arcpy.copy(crop_template_path, crop_path) # Remove extra fields # for field in _arcpy.list_fields(crop_path): # if field not in keep_field_list: # # logging.debug(' {}'.format(field)) # _arcpy.delete_field(crop_path, field) # Add alfalfa cutting field if crop_num in [1, 2, 3, 4]: if dairy_cutting_field not in _arcpy.list_fields(crop_path): logging.debug(' Add field: {}'.format(dairy_cutting_field)) _arcpy.add_field(crop_path, dairy_cutting_field, ogr.OFTInteger) _arcpy.calculate_field(crop_path, dairy_cutting_field, str(dairy_cuttings)) if beef_cutting_field not in _arcpy.list_fields(crop_path): logging.debug(' Add field: {}'.format(beef_cutting_field)) _arcpy.add_field(crop_path, beef_cutting_field, ogr.OFTInteger) _arcpy.calculate_field(crop_path, beef_cutting_field, str(beef_cuttings)) # Write default crop parameters to file # Note: Couldn't use _arcpy.udpate_cursor directly since the # crop_acreage_dict is keyed by crop_num then by cell_id (not FID first) input_driver = _arcpy.get_ogr_driver(crop_path) input_ds = input_driver.Open(crop_path, 1) input_lyr = input_ds.GetLayer() for input_ftr in input_lyr: cell_id = input_ftr.GetField( input_ftr.GetFieldIndex(cell_id_field)) # Don't remove zero acreage crops if in add list if crop_num in crop_add_list: pass # Skip and/or remove zones without crop acreage elif crop_acreage_dict[crop_num][cell_id] < area_threshold: if remove_empty_flag: input_lyr.DeleteFeature(input_ftr.GetFID()) continue # Write parameter values for param_field, param_method, param_type in param_list: input_ftr.SetField(input_ftr.GetFieldIndex(param_field), getattr(crop_param, param_method)) # Write crop acreage if crop_num not in crop_add_list: input_ftr.SetField(input_ftr.GetFieldIndex(crop_acres_field), crop_acreage_dict[crop_num][cell_id]) input_lyr.SetFeature(input_ftr) input_ds = None
def main(ini_path, zone_type='huc8', area_threshold=10, dairy_cuttings=5, beef_cuttings=4, crop_str='', remove_empty_flag=True, overwrite_flag=False): """Build a feature class for each crop and set default crop parameters Apply the values in the CropParams.txt as defaults to every cell Args: ini_path (str): file path of the project INI file zone_type (str): Zone type (huc8, huc10, county, gridmet) area_threshold (float): CDL area threshold [acres] dairy_cuttings (int): Initial number of dairy hay cuttings beef_cuttings (int): Initial number of beef hay cuttings crop_str (str): comma separated list or range of crops to compare overwrite_flag (bool): If True, overwrite existing output rasters Returns: None """ logging.info('\nCalculating ET-Demands Spatial Crop Parameters') remove_empty_flag = True # Input paths # DEADBEEF - For now, get cropET folder from INI file # This function may eventually be moved into the main cropET code crop_et_sec = 'CROP_ET' config = util.read_ini(ini_path, section=crop_et_sec) try: project_ws = config.get(crop_et_sec, 'project_folder') except: logging.error('project_folder parameter must be set in the INI file, ' 'exiting') return False try: gis_ws = config.get(crop_et_sec, 'gis_folder') except: logging.error('gis_folder parameter must be set in the INI file, ' 'exiting') return False try: cells_path = config.get(crop_et_sec, 'cells_path') except: # cells_path = os.path.join(gis_ws, 'ETCells.shp') logging.error('et_cells_path parameter must be set in the INI file, ' 'exiting') return False try: stations_path = config.get(crop_et_sec, 'stations_path') except: logging.error('stations_path parameter must be set in the INI file, ' 'exiting') return False crop_et_ws = config.get(crop_et_sec, 'crop_et_folder') bin_ws = os.path.join(crop_et_ws, 'bin') try: calibration_ws = config.get(crop_et_sec, 'spatial_cal_folder') except: calibration_ws = os.path.join(project_ws, 'calibration') # Sub folder names static_ws = os.path.join(project_ws, 'static') # pmdata_ws = os.path.join(project_ws, 'pmdata') crop_params_path = os.path.join(static_ws, 'CropParams.txt') # ET cells field names cell_id_field = 'CELL_ID' cell_name_field = 'CELL_NAME' crop_acres_field = 'CROP_ACRES' # Only keep the following ET Cell fields keep_field_list = [cell_id_field, cell_name_field, 'AG_ACRES'] # keep_field_list = ['CELL_ID', 'STATION_ID', 'HUC8', 'HUC10', 'GRIDMET_ID, # 'COUNTYNAME', 'AG_ACRES'] # keep_field_list = ['FIPS', 'COUNTYNAME'] # Check input folders if not os.path.isdir(crop_et_ws): logging.error('\nERROR: The INI cropET folder does not exist' '\n {}'.format(crop_et_ws)) sys.exit() elif not os.path.isdir(bin_ws): logging.error('\nERROR: The bin workspace does not exist' '\n {}'.format(bin_ws)) sys.exit() elif not os.path.isdir(project_ws): logging.error('\nERROR: The project folder does not exist' '\n {}'.format(project_ws)) sys.exit() elif not os.path.isdir(gis_ws): logging.error('\nERROR: The GIS folder does not exist' '\n {}'.format(gis_ws)) sys.exit() if '.gdb' not in calibration_ws and not os.path.isdir(calibration_ws): os.makedirs(calibration_ws) logging.info('\nGIS Workspace: {}'.format(gis_ws)) logging.info('Project Workspace: {}'.format(project_ws)) logging.info('CropET Workspace: {}'.format(crop_et_ws)) logging.info('Bin Workspace: {}'.format(bin_ws)) logging.info('Calib. Workspace: {}'.format(calibration_ws)) # Check input files if not os.path.isfile(crop_params_path): logging.error('\nERROR: The crop parameters file does not exist' '\n {}'.format(crop_params_path)) sys.exit() elif not os.path.isfile(cells_path): logging.error('\nERROR: The ET Cell shapefile does not exist' '\n {}'.format(cells_path)) sys.exit() elif not os.path.isfile(stations_path) or not arcpy.Exists(stations_path): logging.error('\nERROR: The weather station shapefile does not exist' '\n {}'.format(stations_path)) sys.exit() logging.debug('Crop Params Path: {}'.format(crop_params_path)) logging.debug('ET Cells Path: {}'.format(cells_path)) logging.debug('Stations Path: {}'.format(stations_path)) # For now, only allow calibration parameters in separate shapefiles ext = '.shp' # # Build output geodatabase if necessary # if calibration_ws.endswith('.gdb'): # logging.debug('GDB Path: {}'.format(calibration_ws)) # ext = '' # if arcpy.Exists(calibration_ws) and overwrite_flag: # try: arcpy.Delete_management(calibration_ws) # except: pass # if calibration_ws is not None and not arcpy.Exists(calibration_ws): # arcpy.CreateFileGDB_management( # os.path.dirname(calibration_ws), # os.path.basename(calibration_ws)) # else: # ext = '.shp' # Field Name, Property, Field Type # Property is the string of the CropParameter class property value # It will be used to access the property using getattr dairy_cutting_field = 'Dairy_Cut' beef_cutting_field = 'Beef_Cut' param_list = [ # ['Name', 'name', 'STRING'], # ['ClassNum', 'class_number', 'LONG'], # ['IsAnnual', 'is_annual', 'SHORT'], # ['IrrigFlag', 'irrigation_flag', 'SHORT'], # ['IrrigDays', 'days_after_planting_irrigation', 'LONG'], # ['Crop_FW', 'crop_fw', 'LONG'], # ['WinterCov', 'winter_surface_cover_class', 'SHORT'], # ['CropKcMax', 'kc_max', 'FLOAT'], ['MAD_Init', 'mad_initial', 'LONG'], ['MAD_Mid', 'mad_midseason', 'LONG'], # ['RootDepIni', 'rooting_depth_initial', 'FLOAT'], # ['RootDepMax', 'rooting_depth_max', 'FLOAT'], # ['EndRootGrw', 'end_of_root_growth_fraction_time', 'FLOAT'], # ['HeightInit', 'height_initial', 'FLOAT'], # ['HeightMax', 'height_max', 'FLOAT'], # ['CurveNum', 'curve_number', 'LONG'], # ['CurveName', 'curve_name', 'STRING'], # ['CurveType', 'curve_type', 'SHORT'], # ['PL_GU_Flag', 'flag_for_means_to_estimate_pl_or_gu', 'SHORT'], ['T30_CGDD', 't30_for_pl_or_gu_or_cgdd', 'FLOAT'], ['PL_GU_Date', 'date_of_pl_or_gu', 'FLOAT'], ['CGDD_Tbase', 'tbase', 'FLOAT'], ['CGDD_EFC', 'cgdd_for_efc', 'LONG'], ['CGDD_Term', 'cgdd_for_termination', 'LONG'], ['Time_EFC', 'time_for_efc', 'LONG'], ['Time_Harv', 'time_for_harvest', 'LONG'], ['KillFrostC', 'killing_frost_temperature', 'FLOAT'], # ['InvokeStrs', 'invoke_stress', 'SHORT'], # ['CN_Coarse', 'cn_coarse_soil', 'LONG'], # ['CN_Medium', 'cn_medium_soil', 'LONG'], # ['CN_Fine', 'cn_fine_soil', 'LONG'] ] # if calibration_ws.endswith('.gdb'): # dairy_cutting_field = 'Dairy_Cuttings' # beef_cutting_field = 'Beef_Cuttings' # param_list = [ # # ['Name', 'name', 'STRING'], # # ['Class_Number', 'class_number', 'LONG'], # # ['Is_Annual', 'is_annual', 'SHORT'], # # ['Irrigation_Flag', 'irrigation_flag', 'SHORT'], # # ['Irrigation_Days', 'days_after_planting_irrigation', 'LONG'], # # ['Crop_FW', 'crop_fw', 'LONG'], # # ['Winter_Cover_Class', 'winter_surface_cover_class', 'SHORT'], # # ['Crop_Kc_Max', 'kc_max', 'FLOAT'], # # ['MAD_Initial', 'mad_initial', 'LONG'], # # ['MAD_Midseason', 'mad_midseason', 'LONG'], # # ['Root_Depth_Ini', 'rooting_depth_initial', 'FLOAT'], # # ['Root_Depth_Max', 'rooting_depth_max', 'FLOAT'], # # ['End_Root_Growth', 'end_of_root_growth_fraction_time', 'FLOAT'], # # ['Height_Initial', 'height_initial', 'FLOAT'], # # ['Height_Maximum', 'height_max', 'FLOAT'], # # ['Curve_Number', 'curve_number', 'LONG'], # # ['Curve_Name', 'curve_name', 'STRING'], # # ['Curve_Type', 'curve_type', 'SHORT'], # # ['PL_GU_Flag', 'flag_for_means_to_estimate_pl_or_gu', 'SHORT'], # ['T30_CGDD', 't30_for_pl_or_gu_or_cgdd', 'FLOAT'], # ['PL_GU_Date', 'date_of_pl_or_gu', 'FLOAT'], # ['CGDD_Tbase', 'tbase', 'FLOAT'], # ['CGDD_EFC', 'cgdd_for_efc', 'LONG'], # ['CGDD_Termination', 'cgdd_for_termination', 'LONG'], # ['Time_EFC', 'time_for_efc', 'LONG'], # ['Time_Harvest', 'time_for_harvest', 'LONG'], # ['Killing_Crost_C', 'killing_frost_temperature', 'FLOAT'], # # ['Invoke_Stress', 'invoke_stress', 'SHORT'], # # ['CN_Coarse_Soil', 'cn_coarse_soil', 'LONG'], # # ['CN_Medium_Soil', 'cn_medium_soil', 'LONG'], # # ['CN_Fine_Soil', 'cn_fine_soil', 'LONG'] # ] crop_add_list = [] if crop_str: try: crop_add_list = sorted(list(util.parse_int_set(crop_str))) # try: # crop_test_list = sorted(list(set( # crop_test_list + list(util.parse_int_set(crop_str))) except: pass # Don't build crop parameter files for non-crops crop_skip_list = sorted(list(set([44, 45, 46, 55, 56, 57]))) # crop_test_list = sorted(list(set(crop_test_list + [46]))) logging.info('\ncrop_add_list = {}'.format(crop_add_list)) # Read crop parameters using ET Demands functions/methods logging.info('\nReading default crop parameters') sys.path.append(bin_ws) import crop_parameters crop_param_dict = crop_parameters.read_crop_parameters(crop_params_path) # arcpy.CheckOutExtension('Spatial') # arcpy.env.pyramid = 'NONE 0' arcpy.env.overwriteOutput = overwrite_flag arcpy.env.parallelProcessingFactor = 8 # Get list of crops specified in ET cells # Currently this may only be crops with CDL acreage crop_field_list = [ field.name for field in arcpy.ListFields(cells_path) if re.match('CROP_\d{2}', field.name)] logging.debug('Cell crop fields: {}'.format(', '.join(crop_field_list))) crop_number_list = [ int(f_name.split('_')[-1]) for f_name in crop_field_list] crop_number_list = [ crop_num for crop_num in crop_number_list if not (crop_skip_list and crop_num in crop_skip_list)] logging.info('Cell crop numbers: {}'.format( ', '.join(list(util.ranges(crop_number_list))))) # Get crop acreages for each cell crop_acreage_dict = defaultdict(dict) field_list = [cell_id_field] + crop_field_list with arcpy.da.SearchCursor(cells_path, field_list) as cursor: for row in cursor: for i, crop_num in enumerate(crop_number_list): # logging.info('{} {}'.format(crop_num, i)) if crop_num in crop_add_list: crop_acreage_dict[crop_num][row[0]] = 0 else: crop_acreage_dict[crop_num][row[0]] = row[i + 1] crop_number_list = sorted(list(set(crop_number_list) | set(crop_add_list))) # Make an empty template crop feature class logging.info('') crop_template_path = os.path.join( calibration_ws, 'crop_00_template' + ext) if overwrite_flag and arcpy.Exists(crop_template_path): logging.debug('Overwriting template crop feature class') arcpy.Delete_management(crop_template_path) if arcpy.Exists(crop_template_path): logging.info('Template crop feature class already exists, skipping') else: logging.info('Building template crop feature class') arcpy.CopyFeatures_management(cells_path, crop_template_path) # Remove unneeded et cell fields for field in arcpy.ListFields(crop_template_path): if (field.name not in keep_field_list and field.editable and not field.required): logging.debug(' Delete field: {}'.format(field.name)) arcpy.DeleteField_management(crop_template_path, field.name) field_list = [f.name for f in arcpy.ListFields(crop_template_path)] # Add crop acreage field if crop_acres_field not in field_list: logging.debug(' Add field: {}'.format(crop_acres_field)) arcpy.AddField_management( crop_template_path, crop_acres_field, 'Float') arcpy.CalculateField_management( crop_template_path, crop_acres_field, '0', 'PYTHON_9.3') # Add crop parameter fields if necessary for param_field, param_method, param_type in param_list: logging.debug(' Add field: {}'.format(param_field)) if param_field not in field_list: arcpy.AddField_management( crop_template_path, param_field, param_type) # if dairy_cutting_field not in field_list: # logging.debug(' Add field: {}'.format(dairy_cutting_field)) # arcpy.AddField_management(crop_template_path, dairy_cutting_field, 'Short') # arcpy.CalculateField_management( # crop_template_path, dairy_cutting_field, dairy_cuttings, 'PYTHON') # if beef_cutting_field not in field_list: # logging.debug(' Add field: {}'.format(beef_cutting_field)) # arcpy.AddField_management(crop_template_path, beef_cutting_field, 'Short') # arcpy.CalculateField_management( # crop_template_path, beef_cutting_field, beef_cuttings, 'PYTHON') # Add an empty/zero crop field for the field mappings below # if len(arcpy.ListFields(cells_path, 'CROP_EMPTY')) == 0: # arcpy.AddField_management(cells_path, 'CROP_EMPTY', 'Float') # arcpy.CalculateField_management( # cells_path, 'CROP_EMPTY', '0', 'PYTHON_9.3') # Process each crop logging.info('\nBuilding crop feature classes') for crop_num in crop_number_list: try: crop_param = crop_param_dict[crop_num] except: continue logging.info('{:>2d} {}'.format(crop_num, crop_param.name)) logging.debug('{}'.format(crop_param)) # Replace other characters with spaces, then remove multiple spaces crop_name = re.sub('[-"().,/~]', ' ', str(crop_param.name).lower()) crop_name = ' '.join(crop_name.strip().split()).replace(' ', '_') crop_path = os.path.join(calibration_ws, 'crop_{0:02d}_{1}{2}'.format( crop_num, crop_name, ext)) # crop_field = 'CROP_{:02d}'.format(crop_num) # Don't check crops in add list if crop_num in crop_add_list: pass # Skip if all zone crop areas are below threshold elif all([v < area_threshold for v in crop_acreage_dict[crop_num].values()]): logging.info(' All crop acreaeges below threshold, skipping crop') continue # Remove existing shapefiles if necessary if overwrite_flag and arcpy.Exists(crop_path): logging.debug(' Overwriting: {}'.format( os.path.basename(crop_path))) arcpy.Delete_management(crop_path) # Don't check skip list until after existing files are removed # if ((crop_test_list and crop_num not in crop_test_list) or # _skip_list and crop_num in crop_skip_list)): # .debug(' Skipping') # Copy ET cells for each crop if needed if arcpy.Exists(crop_path): logging.debug(' Shapefile already exists, skipping') continue else: # logging.debug(' {}'.format(crop_path)) arcpy.Copy_management(crop_template_path, crop_path) # Remove extra fields # for field in arcpy.ListFields(crop_path): # if field.name not in keep_field_list: # # logging.debug(' {}'.format(field.name)) # arcpy.DeleteField_management(crop_path, field.name) # Add alfalfa cutting field if crop_num in [1, 2, 3, 4]: if len(arcpy.ListFields(crop_path, dairy_cutting_field)) == 0: logging.debug(' Add field: {}'.format(dairy_cutting_field)) arcpy.AddField_management( crop_path, dairy_cutting_field, 'Short') arcpy.CalculateField_management( crop_path, dairy_cutting_field, dairy_cuttings, 'PYTHON') if len(arcpy.ListFields(crop_path, beef_cutting_field)) == 0: logging.debug(' Add field: {}'.format(beef_cutting_field)) arcpy.AddField_management( crop_path, beef_cutting_field, 'Short') arcpy.CalculateField_management( crop_path, beef_cutting_field, beef_cuttings, 'PYTHON') # Write default crop parameters to file field_list = [p[0] for p in param_list] + [cell_id_field, crop_acres_field] with arcpy.da.UpdateCursor(crop_path, field_list) as cursor: for row in cursor: # Don't remove zero acreage crops if in add list if crop_num in crop_add_list: pass # Skip and/or remove zones without crop acreage elif crop_acreage_dict[crop_num][row[-2]] < area_threshold: if remove_empty_flag: cursor.deleteRow() continue # Write parameter values for i, (param_field, param_method, param_type) in enumerate(param_list): row[i] = getattr(crop_param, param_method) # Write crop acreage if crop_num not in crop_add_list: row[-1] = crop_acreage_dict[crop_num][row[-2]] cursor.updateRow(row)
def main(gis_ws, cdl_ws, cdl_year, study_area_path, study_area_buffer=None, overwrite_flag=False, pyramids_flag=False, stats_flag=False): """Build study area raster from a target extent and rebuild color table Args: gis_ws (str): Folder/workspace path of the GIS data for the project cdl_ws (str): Folder/workspace path of the GIS data for the project cdl_year (str): Cropland Data Layer year zone_path (str): File path to study area shapefile zone_buffer (float): Distance to buffer input extent Units will be the same as the extent spatial reference overwrite_flag (bool): If True, overwrite output raster pyramids_flag (bool): If True, build pyramids/overviews for the output raster stats_flag (bool): If True, compute statistics for the output raster Returns: None """ scratch_ws = os.path.join(gis_ws, 'scratch') zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img') zone_polygon_path = os.path.join(scratch_ws, 'zone_polygon.shp') # If multiple years were passed in, only use the first one cdl_year = list(util.parse_int_set(cdl_year))[0] cdl_format = '{}_30m_cdls.img' cdl_path = os.path.join(cdl_ws, cdl_format.format(cdl_year)) # Reference all output rasters to CDL # output_osr = gdc.raster_path_osr(cdl_path) output_proj = gdc.raster_path_proj(cdl_path) output_cs = gdc.raster_path_cellsize(cdl_path)[0] output_x, output_y = gdc.raster_path_origin(cdl_path) # output_osr = gdc.proj4_osr( # "+proj=aea +lat_1=29.5 +lat_2=45.5 +lat_0=23 +lon_0=-96 "+ # "+x_0=0 +y_0=0 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m "+ # "+no_defs") # output_cs = 30 # output_x, output_y = 15, 15 output_format = 'HFA' if pyramids_flag: levels = '2 4 8 16 32 64 128' # gdal.SetConfigOption('USE_RRD', 'YES') # gdal.SetConfigOption('HFA_USE_RRD', 'YES') # gdal.SetConfigOption('HFA_COMPRESS_OVR', 'YES') if os.name == 'posix': shell_flag = False else: shell_flag = True # Check input folders if not os.path.isdir(gis_ws): logging.error('\nERROR: The GIS workspace does not exist' '\n {}'.format(gis_ws)) sys.exit() elif not os.path.isfile(cdl_path): logging.error('\nERROR: The input CDL raster does not exist' '\n {}'.format(cdl_path)) sys.exit() elif not os.path.isfile(study_area_path): logging.error('\nERROR: The extent shapefile does not exist' '\n {}'.format(study_area_path)) sys.exit() if not os.path.isdir(scratch_ws): os.makedirs(scratch_ws) logging.info('\nGIS Workspace: {}'.format(gis_ws)) logging.info('Scratch Workspace: {}'.format(scratch_ws)) # Overwrite if os.path.isfile(zone_raster_path) and overwrite_flag: subprocess.check_output( ['gdalmanage', 'delete', '-f', output_format, zone_raster_path], shell=shell_flag) if os.path.isfile(zone_polygon_path) and overwrite_flag: remove_file(zone_polygon_path) # subprocess.check_output( # ['gdalmanage', 'delete', zone_polygon_path], # shell=shell_flag) # Project extent shapefile to CDL spatial reference if not os.path.isfile(zone_polygon_path): # Project study area extent to the input/CDL spatial reference logging.info('Projecting extent shapefile') subprocess.check_output([ 'ogr2ogr', '-f', 'ESRI Shapefile', '-overwrite', '-preserve_fid', '-unsetFieldWidth', '-t_srs', str(output_proj), zone_polygon_path, study_area_path ], shell=shell_flag) # Get the study area extent from the projected shapefile clip_extent = gdc.feature_path_extent(zone_polygon_path) logging.debug('Clip Extent: {}'.format(clip_extent)) # This will buffer in the CDL spatial reference & units if study_area_buffer is not None: logging.debug('Buffering: {}'.format(study_area_buffer)) clip_extent.buffer(study_area_buffer) logging.debug('Clip Extent: {}'.format(clip_extent)) clip_extent.adjust_to_snap(output_x, output_y, output_cs, method='EXPAND') logging.debug('Clip Extent: {}'.format(clip_extent)) # gdal_translate uses ul/lr corners, not extent clip_ullr = clip_extent.ul_lr_swap() logging.debug('Clip UL/LR: {}'.format(clip_ullr)) # Rasterize extent shapefile for masking in other scripts if (not os.path.isfile(zone_raster_path) and os.path.isfile(zone_polygon_path)): logging.info('Rasterizing shapefile') subprocess.check_output([ 'gdal_rasterize', '-of', output_format, '-ot', 'Byte', '-burn', '1', '-init', '0', '-a_nodata', '255', '-co', 'COMPRESSED=YES' ] + ['-te'] + str(clip_extent).split() + [ '-tr', str(output_cs), str(output_cs), zone_polygon_path, zone_raster_path ], shell=shell_flag) # remove_file(zonse_polygon_path) # Statistics if stats_flag and os.path.isfile(zone_raster_path): logging.info('Computing statistics') logging.debug(' {}'.format(zone_raster_path)) subprocess.check_output([ 'gdalinfo', '-stats', '-nomd', '-noct', '-norat', zone_raster_path ], shell=shell_flag) # Pyramids if pyramids_flag and os.path.isfile(zone_raster_path): logging.info('Building statistics') logging.debug(' {}'.format(zone_raster_path)) subprocess.check_output(['gdaladdo', '-ro', zone_raster_path] + levels.split(), shell=shell_flag)
def main(gis_ws, cdl_year='', block_size=16384, mask_flag=False, overwrite_flag=False, pyramids_flag=False, stats_flag=False, agland_nodata=0, agmask_nodata=0): """Mask CDL values for non-agricultural pixels Use CDL derived agmask (in CDL workspace) to define agricultural pixels Args: gis_ws (str): Folder/workspace path of the GIS data for the project cdl_year (str): Comma separated list and/or range of years block_size (int): Maximum block size to use for raster processing mask_flag (bool): If True, mask pixels outside extent shapefile overwrite_flag (bool): If True, overwrite output rasters pyramids_flag (bool): If True, build pyramids/overviews for the output rasters stats_flag (bool): If True, compute statistics for the output rasters agland_nodata: Integer of the nodata value in the agland raster agmask_nodata: Integer of the nodata value in the agmask raster Returns: None """ logging.info('\nExtracting Agriculatural CDL Values') cdl_format = '{0}_30m_cdls.img' cdl_ws = os.path.join(gis_ws, 'cdl') scratch_ws = os.path.join(gis_ws, 'scratch') zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img') output_format = 'HFA' # Ag landuses are 1, all others in state are 0, outside state is nodata # Crop 61 is fallow/idle and was excluded from analysis # Crop 176 is Grassland/Pasture in the new national CDL rasters # Crop 181 was Pasture/Hay in the old state CDL rasters # Crop 182 was Cultivated Crop in the old state CDL rasters agmask_remap = [ [1, 60, 1], [61, 65, 0], # [1, 61, 1], [62, 65, 0], [66, 80, 1], [81, 92, 0], # [93, 180, 0], [181, 182, 1], [183, 203, 0], [204, 254, 1]] # [93, 180, 0], [176, 1], [183, 203, 0], [204, 254, 1]] [97, 100, 1], [101, 203, 0], [204, 254, 1]] if pyramids_flag: levels = '2 4 8 16 32 64 128' # gdal.SetConfigOption('USE_RRD', 'YES') # gdal.SetConfigOption('HFA_USE_RRD', 'YES') # gdal.SetConfigOption('HFA_COMPRESS_OVR', 'YES') if os.name == 'posix': shell_flag = False else: shell_flag = True # Check input folders if not os.path.isdir(gis_ws): logging.error('\nERROR: The GIS workspace does not exist' '\n {}'.format(gis_ws)) sys.exit() elif not os.path.isdir(cdl_ws): logging.error('\nERROR: The CDL workspace does not exist' '\n {}'.format(cdl_ws)) sys.exit() elif mask_flag and not os.path.isfile(zone_raster_path): logging.error( '\nERROR: The zone raster {} does not exist\n' ' Try re-running "clip_cdl_raster.py"'.format(zone_raster_path)) sys.exit() logging.info('\nGIS Workspace: {}'.format(gis_ws)) logging.info('CDL Workspace: {}'.format(cdl_ws)) # Process each CDL year separately for cdl_year in list(util.parse_int_set(cdl_year)): logging.info('\n{}'.format(cdl_year)) cdl_path = os.path.join(cdl_ws, cdl_format.format(cdl_year)) agmask_path = os.path.join( cdl_ws, 'agmask_{}_30m_cdls.img'.format(cdl_year)) agland_path = os.path.join( cdl_ws, 'agland_{}_30m_cdls.img'.format(cdl_year)) if not os.path.isfile(cdl_path): logging.error('\nERROR: The CDL raster does not exist' '\n {}'.format(cdl_path)) continue # Get color table and spatial reference from CDL raster logging.info('Reading CDL color table') cdl_raster_ds = gdal.Open(cdl_path, 0) cdl_geo = gdc.raster_ds_geo(cdl_raster_ds) cdl_rows, cdl_cols = gdc.raster_ds_shape(cdl_raster_ds) cdl_extent = gdc.geo_extent(cdl_geo, cdl_rows, cdl_cols) cdl_proj = gdc.raster_ds_proj(cdl_raster_ds) # DEADBEEF - Why is this hardcoded? # cdl_cellsize = 30 cdl_cellsize = gdc.raster_ds_cellsize(cdl_raster_ds)[0] # cdl_band = cdl_raster_ds.GetRasterBand(1) # cdl_rat = cdl_band.GetDefaultRAT() # cdl_classname_dict = dict() # for row_i in range(cdl_rat.GetRowCount()): # cdl_classname_dict[row_i] = cdl_rat.GetValueAsString( # row_i, cdl_rat.GetColOfUsage(2)) # cdl_raster_ds = None # del cdl_raster_ds, cdl_band, cdl_rat # Copy the input raster to hold the ag data logging.debug('{}'.format(agland_path)) if os.path.isfile(agland_path) and overwrite_flag: subprocess.check_output( ['gdalmanage', 'delete', '-f', output_format, agland_path], shell=shell_flag) if not os.path.isfile(agland_path): logging.info('Copying CDL raster') logging.debug('{}'.format(cdl_path)) subprocess.check_output( ['gdal_translate', '-of', output_format, '-co', 'COMPRESSED=YES', cdl_path, agland_path], shell=shell_flag) # '-a_nodata', agland_nodata if os.path.isfile(cdl_path.replace('.img', '.img.vat.dbf')): shutil.copyfile( cdl_path.replace('.img', '.img.vat.dbf'), agland_path.replace('.img', '.img.vat.dbf') ) # Set the nodata value after copying agland_ds = gdal.Open(agland_path, 1) agland_band = agland_ds.GetRasterBand(1) agland_band.SetNoDataValue(agland_nodata) agland_ds = None # # Get the colormap from the input CDL raster # logging.debug('Re-building raster attribute tables') # agland_ds = gdal.Open(agland_path, 1) # agland_band = agland_ds.GetRasterBand(1) # agland_rat = agland_band.GetDefaultRAT() # agland_usage_list = [ # agland_rat.GetUsageOfCol(col_i) # for col_i in range(agland_rat.GetColumnCount())] # # if 1 not in agland_usage_list: # # _rat.CreateColumn('Count', 1, 1) # # WHY IS THE LOOKING AT Class_Name??? -CHRIS # if 2 not in agland_usage_list: # agland_rat.CreateColumn('Class_Name', 2, 2) # for row_i in range(agland_rat.GetRowCount()): # agland_rat.SetValueAsString( # row_i, agland_rat.GetColOfUsage(2), # cdl_classname_dict[row_i]) # agland_band.SetDefaultRAT(agland_rat) # agland_ds = None # Build an empty output raster to hold the ag mask logging.info('\nBuilding empty ag mask raster') logging.debug('{}'.format(agmask_path)) if os.path.isfile(agmask_path) and overwrite_flag: subprocess.check_output( ['gdalmanage', 'delete', '-f', output_format, agmask_path], shell=shell_flag) if not os.path.isfile(agmask_path): gdc.build_empty_raster( agmask_path, band_cnt=1, output_dtype=np.uint8, output_nodata=agmask_nodata, output_proj=cdl_proj, output_cs=cdl_cellsize, output_extent=cdl_extent) # Set the nodata value after initializing agmask_ds = gdal.Open(agmask_path, 1) agmask_band = agmask_ds.GetRasterBand(1) agmask_band.SetNoDataValue(agmask_nodata) agmask_ds = None # Set non-ag areas to nodata value logging.info('\nProcessing by block') logging.debug(' Input cols/rows: {0}/{1}'.format(cdl_cols, cdl_rows)) for b_i, b_j in gdc.block_gen(cdl_rows, cdl_cols, block_size): logging.info(' Block y: {0:5d} x: {1:5d}'.format(b_i, b_j)) # Read in data for block cdl_array = gdc.raster_to_block( cdl_path, b_i, b_j, block_size, fill_value=0, return_nodata=False) cdl_mask = np.zeros(cdl_array.shape, dtype=np.bool) remap_mask = np.zeros(cdl_array.shape, dtype=np.bool) # Mask CDL values outside extent shapefile if mask_flag and os.path.isfile(zone_raster_path): zone_array = gdc.raster_to_block( zone_raster_path, b_i, b_j, block_size) cdl_array[zone_array == 0] = 0 # Reclassify to 1 for ag and 0 for non-ag for [start, end, value] in agmask_remap: if value == 0: continue logging.debug([start, end, value]) remap_mask |= (cdl_array >= start) & (cdl_array <= end) cdl_mask[remap_mask] = True del remap_mask # Set non-ag areas in agmask to nodata cdl_mask[~cdl_mask] = agmask_nodata # Set non-ag areas in aglands to nodata cdl_array[~cdl_mask] = agland_nodata gdc.block_to_raster(cdl_array, agland_path, b_i, b_j, block_size) gdc.block_to_raster(cdl_mask.astype(np.uint8), agmask_path, b_i, b_j, block_size) del cdl_array, cdl_mask if stats_flag: logging.info('Computing statistics') if os.path.isfile(agland_path): logging.debug('{}'.format(agland_path)) subprocess.check_output( ['gdalinfo', '-stats', '-nomd', '-noct', '-norat', agland_path], shell=shell_flag) if os.path.isfile(agmask_path): logging.debug('{}'.format(agmask_path)) subprocess.check_output( ['gdalinfo', '-stats', '-nomd', '-noct', '-norat', agmask_path], shell=shell_flag) if pyramids_flag: logging.info('Building pyramids') if os.path.isfile(agland_path): logging.debug('{}'.format(agland_path)) subprocess.check_output( ['gdaladdo', '-ro', agland_path] + levels.split(), shell=shell_flag) # args = ['gdaladdo', '-ro'] # if agland_path.endswith('.img'): # args.extend([ # '--config', 'USE_RRD YES', # '--config', 'HFA_USE_RRD YES', # '--config', 'HFA_COMPRESS_OVR YES']) # args.append(agland_path) # args.extend(levels.split()) # subprocess.check_output(args, shell=shell_flag) if os.path.isfile(agmask_path): logging.debug('{}'.format(agmask_path)) subprocess.check_output( ['gdaladdo', '-ro', agmask_path] + levels.split(), shell=shell_flag)