def main(gis_ws,
cdl_year='',
block_size=16384,
mask_flag=False,
overwrite_flag=False,
pyramids_flag=False,
stats_flag=False):
"""Mask DEM values for non-agricultural pixels
Use CDL derived agmask (in CDL workspace) to define agricultural pixels
Args:
gis_ws (str): Folder/workspace path of the GIS data for the project
cdl_year (str): Comma separated list and/or range of years
block_size (int): Maximum block size to use for raster processing
mask_flag (bool): If True, mask pixels outside extent shapefile
overwrite_flag (bool): If True, overwrite existing files
pyramids_flag (bool): If True, build pyramids/overviews
for the output rasters
stats_flag (bool): If True, compute statistics for the output rasters
Returns:
None
"""
logging.info('\nExtracting Agriculatural DEM Values')
input_dem_name = 'ned_30m_albers.img'
cdl_format = '{}_30m_cdls.img'
dem_ws = os.path.join(gis_ws, 'dem')
cdl_ws = os.path.join(gis_ws, 'cdl')
scratch_ws = os.path.join(gis_ws, 'scratch')
zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img')
output_format = 'HFA'
if pyramids_flag:
levels = '2 4 8 16 32 64 128'
# gdal.SetConfigOption('USE_RRD', 'YES')
# gdal.SetConfigOption('HFA_USE_RRD', 'YES')
# gdal.SetConfigOption('HFA_COMPRESS_OVR', 'YES')
if os.name == 'posix':
shell_flag = False
else:
shell_flag = True
# Check input folders
if not os.path.isdir(gis_ws):
logging.error('\nERROR: The GIS workspace does not exist'
'\n {}'.format(gis_ws))
sys.exit()
elif not os.path.isdir(cdl_ws):
logging.error('\nERROR: The CDL workspace does not exist'
'\n {}'.format(cdl_ws))
sys.exit()
elif not os.path.isdir(dem_ws):
logging.error('\nERROR: The DEM workspace does not exist'
'\n {}'.format(dem_ws))
sys.exit()
elif mask_flag and not os.path.isfile(zone_raster_path):
logging.error(
'\nERROR: The zone raster {} does not exist\n'
' Try re-running "clip_cdl_raster.py"'.format(zone_raster_path))
sys.exit()
logging.info('\nGIS Workspace: {}'.format(gis_ws))
logging.info('CDL Workspace: {}'.format(cdl_ws))
logging.info('DEM Workspace: {}\n'.format(dem_ws))
# Check input files
input_dem_path = os.path.join(dem_ws, input_dem_name)
if not os.path.isfile(input_dem_path):
logging.error(
'\nERROR: The raster {} does not exist'.format(input_dem_path))
sys.exit()
# Process existing dem rasters (from merge_dems.py)
input_rows, input_cols = gdc.raster_path_shape(input_dem_path)
# Process each CDL year separately
for cdl_year in list(util.parse_int_set(cdl_year)):
logging.info('\n{}'.format(cdl_year))
# cdl_path = os.path.join(cdl_ws, cdl_format.format(cdl_year))
output_dem_path = os.path.join(dem_ws,
'dem_{}_30m_cdls.img'.format(cdl_year))
# agland_path = os.path.join(
# cdl_ws, 'agland_{}_30m_cdls.img'.format(cdl_year))
agmask_path = os.path.join(cdl_ws,
'agmask_{}_30m_cdls.img'.format(cdl_year))
if not os.path.isfile(agmask_path):
logging.error('\nERROR: The ag-mask raster {} does not exist\n'
' Try re-running "build_ag_cdl_rasters.py"'.format(
agmask_path))
continue
# Copy input DEM
if overwrite_flag and os.path.isfile(output_dem_path):
subprocess.check_output(
['gdalmanage', 'delete', '-f', output_format, output_dem_path],
shell=shell_flag)
if (os.path.isfile(input_dem_path)
and not os.path.isfile(output_dem_path)):
logging.info('\nCopying DEM raster')
logging.debug('{}'.format(input_dem_path))
subprocess.check_output([
'gdal_translate', '-of', output_format, '-co',
'COMPRESSED=YES', input_dem_path, output_dem_path
],
shell=shell_flag)
# Set non-ag areas to nodata value
logging.debug('Processing by block')
logging.debug(' Input cols/rows: {0}/{1}'.format(
input_cols, input_rows))
for b_i, b_j in gdc.block_gen(input_rows, input_cols, block_size):
logging.debug(' Block y: {0:5d} x: {1:5d}'.format(b_i, b_j))
# Read in data for block
agmask_array = gdc.raster_to_block(agmask_path,
b_i,
b_j,
block_size,
return_nodata=False)
dem_array, dem_nodata = gdc.raster_to_block(input_dem_path,
b_i,
b_j,
block_size,
return_nodata=True)
# Mask CDL values outside extent shapefile
if mask_flag and os.path.isfile(zone_raster_path):
zone_array = gdc.raster_to_block(zone_raster_path, b_i, b_j,
block_size)
dem_array[zone_array == 0] = dem_nodata
# Set dem values for non-ag pixels to nodata
dem_array[~agmask_array.astype(np.bool)] = dem_nodata
gdc.block_to_raster(dem_array, output_dem_path, b_i, b_j,
block_size)
del agmask_array, dem_array, dem_nodata
if stats_flag and os.path.isfile(output_dem_path):
logging.info('Computing statistics')
logging.debug(' {}'.format(output_dem_path))
subprocess.check_output(
['gdalinfo', '-stats', '-nomd', output_dem_path],
shell=shell_flag)
if pyramids_flag and os.path.isfile(output_dem_path):
logging.info('Building pyramids')
logging.debug(' {}'.format(output_dem_path))
subprocess.check_output(['gdaladdo', '-ro', output_dem_path] +
levels.split(),
shell=shell_flag)
def main(gis_ws,
input_soil_ws,
cdl_year='',
prop_list=['all'],
block_size=16384,
mask_flag=False,
overwrite_flag=False,
pyramids_flag=False,
stats_flag=False):
"""Mask soil values for non-agricultural pixels
Use CDL derived agmask (in CDL workspace) to define agricultural pixels
Args:
gis_ws (str): Folder/workspace path of the GIS data for the project
input_soil_ws (str): Folder/workspace path of the common soils data
cdl_year (str): Comma separated list and/or range of years
prop_list (list): String of the soil types to build
(i.e. awc, clay, sand, all)
block_size (int): Maximum block size to use for raster processing
mask_flag (bool): If True, mask pixels outside extent shapefile
overwrite_flag (bool): If True, overwrite existing files
pyramids_flag (bool): If True, build pyramids/overviews
for the output rasters
stats_flag (bool): If True, compute statistics for the output rasters
Returns:
None
"""
logging.info('\nExtracting Agriculatural Soil Values')
input_soil_fmt = '{}_30m_albers.img'
# cdl_format = '{0}_30m_cdls.img'
cdl_ws = os.path.join(gis_ws, 'cdl')
# input_soil_ws = os.path.join(gis_ws, 'statsgo')
output_soil_ws = os.path.join(gis_ws, 'soils')
scratch_ws = os.path.join(gis_ws, 'scratch')
zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img')
output_format = 'HFA'
output_nodata = -9999
if pyramids_flag:
levels = '2 4 8 16 32 64 128'
# gdal.SetConfigOption('USE_RRD', 'YES')
# gdal.SetConfigOption('HFA_USE_RRD', 'YES')
# gdal.SetConfigOption('HFA_COMPRESS_OVR', 'YES')
if os.name == 'posix':
shell_flag = False
else:
shell_flag = True
# Check input folders
if not os.path.isdir(gis_ws):
logging.error('\nERROR: The GIS folder does not exist'
'\n {}'.format(gis_ws))
sys.exit()
elif not os.path.isdir(cdl_ws):
logging.error('\nERROR: The CDL folder does not exist'
'\n {}'.format(cdl_ws))
sys.exit()
elif not os.path.isdir(input_soil_ws):
logging.error('\nERROR: The input soil folder does not exist'
'\n {}'.format(input_soil_ws))
sys.exit()
elif mask_flag and not os.path.isfile(zone_raster_path):
logging.error(
'\nERROR: The zone raster {} does not exist\n'
' Try re-running "clip_cdl_raster.py"'.format(zone_raster_path))
sys.exit()
logging.info('\nGIS Workspace: {}'.format(gis_ws))
logging.info('CDL Workspace: {}'.format(cdl_ws))
logging.info('Input Soil Workspace: {}'.format(input_soil_ws))
logging.info('Output Soil Workspace: {}'.format(output_soil_ws))
# Process each CDL year separately
for cdl_year in list(util.parse_int_set(cdl_year)):
logging.info('\n{}'.format(cdl_year))
# cdl_path = os.path.join(cdl_ws, cdl_format.format(cdl_year))
output_soil_fmt = '{}_{}_30m_cdls.img'.format('{}', cdl_year)
# agland_path = os.path.join(
# cdl_ws, 'agland_{}_30m_cdls.img'.format(cdl_year))
agmask_path = os.path.join(cdl_ws,
'agmask_{}_30m_cdls.img'.format(cdl_year))
if not os.path.isfile(agmask_path):
logging.error('\nERROR: The ag-mask raster {} does not exist\n' +
' Try re-running "build_ag_cdl_rasters.py"'.format(
agmask_path))
continue
logging.info('Soil Property: {}'.format(', '.join(prop_list)))
if prop_list == ['all']:
prop_list = ['AWC', 'CLAY', 'SAND']
# Process existing soil rasters (from rasterize script)
for prop_str in prop_list:
logging.info('\nSoil: {}'.format(prop_str.upper()))
input_soil_path = os.path.join(input_soil_ws,
input_soil_fmt.format(prop_str))
output_soil_path = os.path.join(
output_soil_ws, output_soil_fmt.format(prop_str.lower()))
if not os.path.isfile(input_soil_path):
logging.error('\nERROR: The raster {} does not exist'.format(
input_soil_path))
continue
# Create a copy of the input raster to modify
if overwrite_flag and os.path.isfile(output_soil_path):
subprocess.check_output([
'gdalmanage', 'delete', '-f', output_format,
output_soil_path
],
shell=shell_flag)
if (os.path.isfile(input_soil_path)
and not os.path.isfile(output_soil_path)):
logging.info('\nCopying soil raster')
logging.debug('{}'.format(input_soil_path))
subprocess.check_output([
'gdal_translate', '-of', output_format, '-co',
'COMPRESSED=YES', input_soil_path, output_soil_path
],
shell=shell_flag)
# Get the size of the input raster
input_rows, input_cols = gdc.raster_path_shape(input_soil_path)
# Set non-ag areas to nodata value
logging.debug('Processing by block')
logging.debug(' Input cols/rows: {0}/{1}'.format(
input_cols, input_rows))
for b_i, b_j in gdc.block_gen(input_rows, input_cols, block_size):
logging.debug(' Block y: {0:5d} x: {1:5d}'.format(b_i, b_j))
# Read in data for block
agmask_array = gdc.raster_to_block(agmask_path,
b_i,
b_j,
block_size,
return_nodata=False)
soil_array, soil_nodata = gdc.raster_to_block(
input_soil_path, b_i, b_j, block_size, return_nodata=True)
# Mask CDL values outside extent shapefile
if mask_flag and os.path.isfile(zone_raster_path):
zone_array = gdc.raster_to_block(zone_raster_path, b_i,
b_j, block_size)
soil_array[zone_array == 0] = output_nodata
# Set soil values for non-ag pixels to nodata
soil_array[~agmask_array.astype(np.bool)] = output_nodata
gdc.block_to_raster(soil_array, output_soil_path, b_i, b_j,
block_size)
del agmask_array, soil_array, soil_nodata
# Override the output raster nodata value
output_ds = gdal.Open(output_soil_path, 1)
output_band = output_ds.GetRasterBand(1)
output_band.SetNoDataValue(output_nodata)
output_ds = None
if stats_flag and os.path.isfile(output_soil_path):
logging.info('Computing statistics')
logging.debug(' {}'.format(output_soil_path))
subprocess.check_output(
['gdalinfo', '-stats', '-nomd', output_soil_path],
shell=shell_flag)
if pyramids_flag and os.path.isfile(output_soil_path):
logging.info('Building pyramids')
logging.debug(' {}'.format(output_soil_path))
subprocess.check_output(['gdaladdo', '-ro', output_soil_path] +
levels.split(),
shell=shell_flag)
def main(ini_path, overwrite_flag=False):
"""Build CDL shapefiles for agricultural pixels
Parameters
----------
ini_path : str
File path of the parameter INI file.
overwrite_flag : bool
If True, overwrite existing shapefile (the default is False).
Returns
-------
None
"""
logging.info('\nBuilding Agricultural CDL Shapefile')
logging.debug('INI: {}'.format(ini_path))
config = util.read_ini(ini_path, section='CROP_ET')
zone_path = config.get('CROP_ET', 'cells_path')
crop_path = config.get('CROP_ET', 'crop_path')
temp_path = crop_path.replace('.shp', '_temp.shp')
cdl_ws = config.get('CROP_ET', 'cdl_folder')
cdl_year = int(config.get('CROP_ET', 'cdl_year'))
cdl_format = config.get('CROP_ET', 'cdl_format')
# It might make more sense to pass the non-ag CDL values instead
cdl_crops = util.parse_int_set(config.get('CROP_ET', 'cdl_crops'))
# cdl_nonag = util.parse_int_set(config.get('CROP_ET', 'cdl_nonag'))
cdl_path = os.path.join(cdl_ws, cdl_format.format(cdl_year, 'img'))
# Output field name in the crops shapefile
crop_field = config.get('CROP_ET', 'crop_field')
shp_driver = ogr.GetDriverByName('ESRI Shapefile')
if os.path.isfile(crop_path):
if overwrite_flag:
shp_driver.DeleteDataSource(crop_path)
else:
return True
if not os.path.isfile(zone_path):
logging.error(
'\nERROR: The ET zone shapefile doesn\'t exist, exiting\n'
' {}'.format(zone_path))
sys.exit()
elif not os.path.isfile(cdl_path):
logging.error('\nERROR: The CDL raster doesn\'t exist, exiting\n'
' {}'.format(cdl_path))
sys.exit()
logging.debug('Zones: {}'.format(zone_path))
# CDL Raster Properties
cdl_ds = gdal.Open(cdl_path)
cdl_band = cdl_ds.GetRasterBand(1)
try:
cdl_nodata = int(cdl_band.GetNoDataValue())
except TypeError:
cdl_nodata = 0
cdl_gtype = cdl_band.DataType
cdl_proj = cdl_ds.GetProjection()
cdl_osr = gdc.proj_osr(cdl_proj)
cdl_geo = cdl_ds.GetGeoTransform()
cdl_x, cdl_y = gdc.geo_origin(cdl_geo)
cdl_cs = gdc.geo_cellsize(cdl_geo, x_only=True)
cdl_extent = gdc.raster_ds_extent(cdl_ds)
logging.debug('\nCDL Raster Properties')
logging.debug(' Geo: {}'.format(cdl_geo))
logging.debug(' Snap: {} {}'.format(cdl_x, cdl_y))
logging.debug(' Cellsize: {}'.format(cdl_cs))
logging.debug(' Nodata: {}'.format(cdl_nodata))
logging.debug(' GDAL Type: {}'.format(cdl_gtype))
logging.debug(' Extent: {}'.format(cdl_extent))
logging.debug(' Projection: {}'.format(cdl_osr.ExportToWkt()))
# logging.debug(' OSR: {}'.format(cdl_osr))
# ET Zones Properties
zone_ds = shp_driver.Open(zone_path, 0)
zone_lyr = zone_ds.GetLayer()
zone_osr = zone_lyr.GetSpatialRef()
zone_wkt = gdc.osr_proj(zone_osr)
zone_extent = gdc.feature_lyr_extent(zone_lyr)
logging.debug('\nET Zones Shapefile Properties')
logging.debug(' Extent: {}'.format(zone_extent))
logging.debug(' Projection: {}'.format(zone_osr.ExportToWkt()))
# logging.debug(' OSR: {}'.format(zones_osr))
if zone_osr.IsGeographic():
logging.error('\nERROR: The ET zones shapefile must be in a projected '
'coordinate system, exiting')
sys.exit()
# Subset/clip properties
# Project the extent to the CDL spatial reference
logging.debug('\nClip Subset')
clip_extent = zone_extent.project(zone_osr, cdl_osr)
logging.debug(' Projected: {}'.format(clip_extent))
# Adjust the clip extent to the CDL snap point and cell size
clip_extent.buffer(10 * cdl_cs)
clip_extent.adjust_to_snap(snap_x=cdl_x,
snap_y=cdl_y,
cs=cdl_cs,
method='EXPAND')
logging.debug(' Snapped: {}'.format(clip_extent))
# Limit the subset extent to the CDL extent
clip_extent.clip(cdl_extent)
logging.debug(' Clipped: {}'.format(clip_extent))
# Compute the clip geotransform and shape
clip_geo = clip_extent.geo(cs=cdl_cs)
clip_rows, clip_cols = clip_extent.shape(cs=cdl_cs)
logging.debug(' Rows/Cols: {} {}'.format(clip_rows, clip_cols))
# Building a raster mask was a little more efficient than selecting
# touching features later on.
logging.debug('\nBuilding ET Zones mask')
zone_count = zone_lyr.GetFeatureCount()
if zone_count < 255:
zone_mask_gtype = gdal.GDT_Byte
zone_mask_nodata = 255
elif zone_count < 65535:
zone_mask_gtype = gdal.GDT_UInt16
zone_mask_nodata = 65535
else:
zone_mask_gtype = gdal.GDT_UInt32
zone_mask_nodata = 4294967295
memory_driver = gdal.GetDriverByName('GTiff')
# zones_mask_ds = memory_driver.Create(
# os.path.join(os.path.dirname(zones_path), 'zones_mask.tiff'),
# clip_cols, clip_rows, 1, zones_mask_gtype)
memory_driver = gdal.GetDriverByName('MEM')
zone_mask_ds = memory_driver.Create('', clip_cols, clip_rows, 1,
zone_mask_gtype)
zone_mask_ds.SetProjection(cdl_proj)
zone_mask_ds.SetGeoTransform(clip_geo)
zone_mask_band = zone_mask_ds.GetRasterBand(1)
zone_mask_band.Fill(zone_mask_nodata)
zone_mask_band.SetNoDataValue(zone_mask_nodata)
gdal.RasterizeLayer(zone_mask_ds, [1], zone_lyr, burn_values=[1])
# zones_mask_ds = None
# zones_mask_band = zones_mask_ds.GetRasterBand(1)
zone_mask = zone_mask_band.ReadAsArray(0, 0, clip_cols, clip_rows)
zone_mask = (zone_mask != zone_mask_nodata)
zone_mask_ds = None
logging.debug('\nBuilding initial CDL polygon shapefile')
if os.path.isfile(temp_path):
shp_driver.DeleteDataSource(temp_path)
polygon_ds = shp_driver.CreateDataSource(temp_path)
polygon_lyr = polygon_ds.CreateLayer('OUTPUT_POLY',
geom_type=ogr.wkbPolygon)
field_defn = ogr.FieldDefn(crop_field, ogr.OFTInteger)
polygon_lyr.CreateField(field_defn)
# TODO: Process CDL by tile
# logging.debug('\nProcessing CDL by tile')
# tile_list = [[0, 0]]
# for tile_i, tile_j in tile_list:
# logging.debug(' Tile: {} {}'.format(tile_i, tile_j))
logging.debug('\nConverting CDL raster to polygon')
# Read the CDL subset array
clip_xi, clip_yi = array_geo_offsets(cdl_geo, clip_geo)
logging.debug(' Subset i/j: {} {}'.format(clip_xi, clip_yi))
cdl_array = cdl_band.ReadAsArray(clip_xi, clip_yi, clip_cols, clip_rows)
cdl_ds = None
# Apply the zones mask
if np.any(zone_mask):
cdl_array[~zone_mask] = cdl_nodata
# Set non-agricultural pixels to nodata
logging.debug('\nMasking non-crop pixels')
cdl_array_values = np.unique(cdl_array)
nodata_mask = np.zeros(cdl_array.shape, dtype=np.bool)
for value in range(1, 255):
if value in cdl_crops:
continue
elif value not in cdl_array_values:
continue
# logging.debug(' Value: {}'.format(value))
nodata_mask |= (cdl_array == value)
cdl_array[nodata_mask] = cdl_nodata
# # DEADBEEF - This is using the remap ranges
# # It is probably more efficient than processing each crop separately
# nodata_mask = np.zeros(cdl_array.shape, dtype=np.bool)
# for [start, end, value] in cdl_agmask_remap:
# if value == 1:
# continue
# logging.debug([start, end, value])
# nodata_mask |= (cdl_array >= start) & (cdl_array <= end)
# cdl_array[nodata_mask] = cdl_nodata
# Create an in-memory raster to read the CDL into
# Set the mask band separately
memory_driver = gdal.GetDriverByName('MEM')
memory_ds = memory_driver.Create('', clip_cols, clip_rows, 2, cdl_gtype)
memory_ds.SetGeoTransform(clip_geo)
memory_ds.SetProjection(cdl_proj)
memory_band = memory_ds.GetRasterBand(1)
memory_band.SetNoDataValue(cdl_nodata)
mask_band = memory_ds.GetRasterBand(2)
# Write the CDL subset array to the memory raster
logging.debug('\nWriting array')
memory_band.WriteArray(cdl_array, 0, 0)
mask_band.WriteArray(cdl_array != cdl_nodata, 0, 0)
# Polygonize the CDL array
logging.debug('\nConverting raster to polygon')
gdal.Polygonize(memory_band, mask_band, polygon_lyr, 0)
# Cleanup
mask_band = None
memory_band = None
memory_ds = None
polygon_lyr = None
polygon_ds = None
del cdl_array, nodata_mask, zone_mask
# Write projection/spatial reference
prj_osr = gdc.proj_osr(cdl_proj)
prj_osr.MorphToESRI()
with open(temp_path.replace('.shp', '.prj'), 'w') as prj_f:
prj_f.write(prj_osr.ExportToWkt())
# Project crops to zones spatial reference
logging.debug('\nProjecting crops to ET zones spatial reference')
# ogr2ogr.project(temp_path, crop_path, zones_wkt)
arcpy.project(temp_path, crop_path, zone_osr)
logging.debug('\nRemoving temporary crops shapefile')
arcpy.delete(temp_path)
def main(ini_path,
area_threshold=10,
dairy_cuttings=5,
beef_cuttings=4,
crop_str='',
overwrite_flag=False):
"""Build a feature class for each crop and set default crop parameters
Apply the values in the CropParams.txt as defaults to every cell
Parameters
----------
ini_path : str
File path of the parameter INI file.
area_threshold : float
CDL area threshold [acres].
dairy_cuttings : int
Initial number of dairy hay cuttings.
beef_cuttings : int
Initial number of beef hay cuttings.
crop_str : str
Comma separated list or range of crops to compare (no spaces, ex: 1,2,4-6)
overwrite_flag : bool
If True, overwrite existing output rasters.
Returns
-------
None
"""
logging.info('\nCalculating ET-Demands Spatial Crop Parameters')
remove_empty_flag = True
# Input paths
# DEADBEEF - For now, get cropET folder from INI file
# This function may eventually be moved into the main cropET code
crop_et_sec = 'CROP_ET'
config = util.read_ini(ini_path, section=crop_et_sec)
try:
project_ws = config.get(crop_et_sec, 'project_folder')
except:
logging.error('project_folder parameter must be set in the INI file, '
'exiting')
return False
try:
gis_ws = config.get(crop_et_sec, 'gis_folder')
except:
logging.error('gis_folder parameter must be set in the INI file, '
'exiting')
return False
try:
cells_path = config.get(crop_et_sec, 'cells_path')
except:
# cells_path = os.path.join(gis_ws, 'ETCells.shp')
logging.error('et_cells_path parameter must be set in the INI file, '
'exiting')
return False
try:
stations_path = config.get(crop_et_sec, 'stations_path')
except:
logging.error('stations_path parameter must be set in the INI file, '
'exiting')
return False
try:
crop_params_name = config.get(crop_et_sec, 'crop_params_name')
except:
logging.error(
'crop_params_name parameter must be set in the INI file, '
'exiting')
return False
crop_et_ws = config.get(crop_et_sec, 'crop_et_folder')
bin_ws = os.path.join(crop_et_ws, 'bin')
try:
calibration_ws = config.get(crop_et_sec, 'spatial_cal_folder')
except:
calibration_ws = os.path.join(project_ws, 'calibration')
# Sub folder names
static_ws = os.path.join(project_ws, 'static')
crop_params_path = os.path.join(static_ws, crop_params_name)
# ET cells field names
cell_id_field = 'CELL_ID'
cell_name_field = 'CELL_NAME'
crop_acres_field = 'CROP_ACRES'
# Only keep the following ET Cell fields
keep_field_list = [cell_id_field, cell_name_field, 'AG_ACRES']
# keep_field_list = ['CELL_ID', 'STATION_ID', 'HUC8', 'HUC10', 'GRIDMET_ID',
# 'COUNTYNAME', 'AG_ACRES']
# keep_field_list = ['FIPS', 'COUNTYNAME']
# Check input folders
if not os.path.isdir(crop_et_ws):
logging.error('\nERROR: The INI cropET folder does not exist'
'\n {}'.format(crop_et_ws))
sys.exit()
elif not os.path.isdir(bin_ws):
logging.error('\nERROR: The bin workspace does not exist'
'\n {}'.format(bin_ws))
sys.exit()
elif not os.path.isdir(project_ws):
logging.error('\nERROR: The project folder does not exist'
'\n {}'.format(project_ws))
sys.exit()
elif not os.path.isdir(gis_ws):
logging.error('\nERROR: The GIS folder does not exist'
'\n {}'.format(gis_ws))
sys.exit()
if '.gdb' not in calibration_ws and not os.path.isdir(calibration_ws):
os.makedirs(calibration_ws)
logging.info('\nGIS Workspace: {}'.format(gis_ws))
logging.info('Project Workspace: {}'.format(project_ws))
logging.info('CropET Workspace: {}'.format(crop_et_ws))
logging.info('Bin Workspace: {}'.format(bin_ws))
logging.info('Calib. Workspace: {}'.format(calibration_ws))
# Check input files
if not os.path.isfile(crop_params_path):
logging.error('\nERROR: The crop parameters file does not exist'
'\n {}'.format(crop_params_path))
sys.exit()
elif not os.path.isfile(cells_path):
logging.error('\nERROR: The ET Cell shapefile does not exist'
'\n {}'.format(cells_path))
sys.exit()
elif not os.path.isfile(stations_path):
logging.error('\nERROR: The weather station shapefile does not exist'
'\n {}'.format(stations_path))
sys.exit()
logging.debug('Crop Params Path: {}'.format(crop_params_path))
logging.debug('ET Cells Path: {}'.format(cells_path))
logging.debug('Stations Path: {}'.format(stations_path))
# For now, only allow calibration parameters in separate shapefiles
ext = '.shp'
# # Build output geodatabase if necessary
# if calibration_ws.endswith('.gdb'):
# logging.debug('GDB Path: {}'.format(calibration_ws))
# ext = ''
# _arcpy.exists(calibration_ws) and overwrite_flag:
# try: _arcpy.delete(calibration_ws)
# except: pass
# if calibration_ws is not None and not _arcpy.exists(calibration_ws):
# arcpy.CreateFileGDB_management(
# os.path.dirname(calibration_ws),
# os.path.basename(calibration_ws))
# else:
# ext = '.shp'
# Field Name, Property, Field Type
# Property is the string of the CropParameter class property value
# It will be used to access the property using getattr
dairy_cutting_field = 'Dairy_Cut'
beef_cutting_field = 'Beef_Cut'
param_list = [
# ['Name', 'name', ogr.OFTString],
# ['ClassNum', 'class_number', ogr.OFTInteger],
# ['IsAnnual', 'is_annual', 'SHORT'],
# ['IrrigFlag', 'irrigation_flag', 'SHORT'],
# ['IrrigDays', 'days_after_planting_irrigation', ogr.OFTInteger],
# ['Crop_FW', 'crop_fw', ogr.OFTInteger],
# ['WinterCov', 'winter_surface_cover_class', 'SHORT'],
# ['CropKcMax', 'kc_max', ogr.OFTReal],
['MAD_Init', 'mad_initial', ogr.OFTInteger],
['MAD_Mid', 'mad_midseason', ogr.OFTInteger],
# ['RootDepIni', 'rooting_depth_initial', ogr.OFTReal],
# ['RootDepMax', 'rooting_depth_max', ogr.OFTReal],
# ['EndRootGrw', 'end_of_root_growth_fraction_time', ogr.OFTReal],
# ['HeightInit', 'height_initial', ogr.OFTReal],
# ['HeightMax', 'height_max', ogr.OFTReal],
# ['CurveNum', 'curve_number', ogr.OFTInteger],
# ['CurveName', 'curve_name', ogr.OFTString],
# ['CurveType', 'curve_type', 'SHORT'],
# ['PL_GU_Flag', 'flag_for_means_to_estimate_pl_or_gu', 'SHORT'],
['T30_CGDD', 't30_for_pl_or_gu_or_cgdd', ogr.OFTReal],
['PL_GU_Date', 'date_of_pl_or_gu', ogr.OFTReal],
['CGDD_Tbase', 'tbase', ogr.OFTReal],
['CGDD_EFC', 'cgdd_for_efc', ogr.OFTInteger],
['CGDD_Term', 'cgdd_for_termination', ogr.OFTInteger],
['Time_EFC', 'time_for_efc', ogr.OFTInteger],
['Time_Harv', 'time_for_harvest', ogr.OFTInteger],
['KillFrostC', 'killing_frost_temperature', ogr.OFTReal],
# ['InvokeStrs', 'invoke_stress', 'SHORT'],
# ['CN_Coarse', 'cn_coarse_soil', ogr.OFTInteger],
# ['CN_Medium', 'cn_medium_soil', ogr.OFTInteger],
# ['CN_Fine', 'cn_fine_soil', ogr.OFTInteger]
]
# if calibration_ws.endswith('.gdb'):
# dairy_cutting_field = 'Dairy_Cuttings'
# beef_cutting_field = 'Beef_Cuttings'
# param_list = [
# # ['Name', 'name', 'STRING'],
# # ['Class_Number', 'class_number', ogr.OFTInteger],
# # ['Is_Annual', 'is_annual', 'SHORT'],
# # ['Irrigation_Flag', 'irrigation_flag', 'SHORT'],
# # ['Irrigation_Days', 'days_after_planting_irrigation', ogr.OFTInteger],
# # ['Crop_FW', 'crop_fw', ogr.OFTInteger],
# # ['Winter_Cover_Class', 'winter_surface_cover_class', 'SHORT'],
# # ['Crop_Kc_Max', 'kc_max', ogr.OFTReal],
# # ['MAD_Initial', 'mad_initial', ogr.OFTInteger],
# # ['MAD_Midseason', 'mad_midseason', ogr.OFTInteger],
# # ['Root_Depth_Ini', 'rooting_depth_initial', ogr.OFTReal],
# # ['Root_Depth_Max', 'rooting_depth_max', ogr.OFTReal],
# # ['End_Root_Growth', 'end_of_root_growth_fraction_time', ogr.OFTReal],
# # ['Height_Initial', 'height_initial', ogr.OFTReal],
# # ['Height_Maximum', 'height_max', ogr.OFTReal],
# # ['Curve_Number', 'curve_number', ogr.OFTInteger],
# # ['Curve_Name', 'curve_name', ogr.OFTString],
# # ['Curve_Type', 'curve_type', 'SHORT'],
# # ['PL_GU_Flag', 'flag_for_means_to_estimate_pl_or_gu', 'SHORT'],
# ['T30_CGDD', 't30_for_pl_or_gu_or_cgdd', ogr.OFTReal],
# ['PL_GU_Date', 'date_of_pl_or_gu', ogr.OFTReal],
# ['CGDD_Tbase', 'tbase', ogr.OFTReal],
# ['CGDD_EFC', 'cgdd_for_efc', ogr.OFTInteger],
# ['CGDD_Termination', 'cgdd_for_termination', ogr.OFTInteger],
# ['Time_EFC', 'time_for_efc', ogr.OFTInteger],
# ['Time_Harvest', 'time_for_harvest', ogr.OFTInteger],
# ['Killing_Crost_C', 'killing_frost_temperature', ogr.OFTReal],
# # ['Invoke_Stress', 'invoke_stress', 'SHORT'],
# # ['CN_Coarse_Soil', 'cn_coarse_soil', ogr.OFTInteger],
# # ['CN_Medium_Soil', 'cn_medium_soil', ogr.OFTInteger],
# # ['CN_Fine_Soil', 'cn_fine_soil', ogr.OFTInteger]
# ]
crop_add_list = []
if crop_str:
try:
crop_add_list = sorted(list(util.parse_int_set(crop_str)))
# try:
# crop_test_list = sorted(list(set(
# crop_test_list + list(util.parse_int_set(crop_str)))
except:
pass
# Don't build crop parameter files for non-crops
crop_skip_list = sorted(list(set([44, 45, 46, 55, 56, 57])))
# crop_test_list = sorted(list(set(crop_test_list + [46])))
logging.info('\ncrop_add_list = {}'.format(crop_add_list))
# Read crop parameters using ET Demands functions/methods
logging.info('\nReading default crop parameters')
sys.path.append(bin_ws)
import crop_parameters
crop_param_dict = crop_parameters.read_crop_parameters(crop_params_path)
# Get list of crops specified in ET cells
# Currently this may only be crops with CDL acreage
crop_field_list = sorted([
field for field in _arcpy.list_fields(cells_path)
if re.match('CROP_\d{2}', field)
])
crop_number_list = [int(f.split('_')[-1]) for f in crop_field_list]
logging.info('Cell crop numbers: {}'.format(', '.join(
list(util.ranges(crop_number_list)))))
logging.debug('Cell crop fields: {}'.format(', '.join(crop_field_list)))
# Get crop acreages for each cell
# DEADBEEF - Does this dict need to be keyed by crop then cell_id?
# Could it be changed to cell_id, crop or fid, crop to make it easier to
# write to the shapefile using update_cursor()?
crop_acreage_dict = defaultdict(dict)
field_list = [cell_id_field] + crop_field_list
for fid, row in _arcpy.search_cursor(cells_path, field_list).items():
for crop_field, crop_num in zip(crop_field_list, crop_number_list):
if crop_skip_list and crop_num in crop_skip_list:
continue
elif crop_num in crop_add_list:
crop_acreage_dict[crop_num][row[cell_id_field]] = 0
elif row[crop_field]:
crop_acreage_dict[crop_num][
row[cell_id_field]] = row[crop_field]
else:
crop_acreage_dict[crop_num][row[cell_id_field]] = 0
crop_number_list = sorted(list(set(crop_number_list) | set(crop_add_list)))
# Make an empty template crop feature class
logging.info('')
crop_template_path = os.path.join(calibration_ws, 'crop_00_template' + ext)
if overwrite_flag and _arcpy.exists(crop_template_path):
logging.debug('Overwriting template crop feature class')
_arcpy.delete(crop_template_path)
if _arcpy.exists(crop_template_path):
logging.info('Template crop feature class already exists, skipping')
else:
logging.info('Building template crop feature class')
_arcpy.copy(cells_path, crop_template_path)
# Remove unneeded et cell fields
for field in _arcpy.list_fields(crop_template_path):
# if (field not in keep_field_list and
# field.editable and not field.required):
if field not in keep_field_list:
logging.debug(' Delete field: {}'.format(field))
_arcpy.delete_field(crop_template_path, field)
field_list = _arcpy.list_fields(crop_template_path)
# Add crop acreage field
if crop_acres_field not in field_list:
logging.debug(' Add field: {}'.format(crop_acres_field))
_arcpy.add_field(crop_template_path, crop_acres_field, ogr.OFTReal)
_arcpy.calculate_field(crop_template_path, crop_acres_field, '0')
# Add crop parameter fields if necessary
for param_field, param_method, param_type in param_list:
logging.debug(' Add field: {}'.format(param_field))
if param_field not in field_list:
_arcpy.add_field(crop_template_path, param_field, param_type)
# if dairy_cutting_field not in field_list:
# logging.debug(' Add field: {}'.format(dairy_cutting_field))
# _arcpy.add_field(crop_template_path, dairy_cutting_field,
# ogr.OFTInteger)
# _arcpy.calculate_field(crop_template_path, dairy_cutting_field,
# dairy_cuttings)
# if beef_cutting_field not in field_list:
# logging.debug(' Add field: {}'.format(beef_cutting_field))
# _arcpy.add_field(crop_template_path, beef_cutting_field,
# ogr.OFTInteger)
# _arcpy.calculate_field(crop_template_path, beef_cutting_field,
# beef_cuttings)
# Add an empty/zero crop field for the field mappings below
# if 'CROP_EMPTY' not in _arcpy.list_fields(cells_path):
# _arcpy.add_field(cells_path, 'CROP_EMPTY', ogr.OFTReal)
# _arcpy.calculate_field(cells_path, 'CROP_EMPTY', '0')
# Process each crop
logging.info('\nBuilding crop feature classes')
for crop_num in crop_number_list:
try:
crop_param = crop_param_dict[crop_num]
except:
continue
logging.info('{:>2d} {}'.format(crop_num, crop_param.name))
logging.debug('{}'.format(crop_param))
# Replace other characters with spaces, then remove multiple spaces
crop_name = re.sub('[-"().,/~]', ' ', str(crop_param.name).lower())
crop_name = ' '.join(crop_name.strip().split()).replace(' ', '_')
crop_path = os.path.join(
calibration_ws,
'crop_{0:02d}_{1}{2}'.format(crop_num, crop_name, ext))
# crop_field = 'CROP_{:02d}'.format(crop_num)
# Don't check crops in add list
if crop_num in crop_add_list:
pass
# Skip if all zone crop areas are below threshold
elif all(
[v < area_threshold
for v in crop_acreage_dict[crop_num].values()]):
logging.info('** Skipping Crop {}, All crop acreages below'
' threshold'.format(crop_num))
continue
# Remove existing shapefiles if necessary
if overwrite_flag and _arcpy.exists(crop_path):
logging.debug(' Overwriting: {}'.format(
os.path.basename(crop_path)))
_arcpy.delete(crop_path)
# Don't check skip list until after existing files are removed
# if ((crop_test_list and crop_num not in crop_test_list) or
# _skip_list and crop_num in crop_skip_list)):
# .debug(' Skipping')
# Copy ET cells for each crop if needed
if _arcpy.exists(crop_path):
logging.debug(' Shapefile already exists, skipping')
continue
else:
# logging.debug(' {}'.format(crop_path))
_arcpy.copy(crop_template_path, crop_path)
# Remove extra fields
# for field in _arcpy.list_fields(crop_path):
# if field not in keep_field_list:
# # logging.debug(' {}'.format(field))
# _arcpy.delete_field(crop_path, field)
# Add alfalfa cutting field
if crop_num in [1, 2, 3, 4]:
if dairy_cutting_field not in _arcpy.list_fields(crop_path):
logging.debug(' Add field: {}'.format(dairy_cutting_field))
_arcpy.add_field(crop_path, dairy_cutting_field,
ogr.OFTInteger)
_arcpy.calculate_field(crop_path, dairy_cutting_field,
str(dairy_cuttings))
if beef_cutting_field not in _arcpy.list_fields(crop_path):
logging.debug(' Add field: {}'.format(beef_cutting_field))
_arcpy.add_field(crop_path, beef_cutting_field, ogr.OFTInteger)
_arcpy.calculate_field(crop_path, beef_cutting_field,
str(beef_cuttings))
# Write default crop parameters to file
# Note: Couldn't use _arcpy.udpate_cursor directly since the
# crop_acreage_dict is keyed by crop_num then by cell_id (not FID first)
input_driver = _arcpy.get_ogr_driver(crop_path)
input_ds = input_driver.Open(crop_path, 1)
input_lyr = input_ds.GetLayer()
for input_ftr in input_lyr:
cell_id = input_ftr.GetField(
input_ftr.GetFieldIndex(cell_id_field))
# Don't remove zero acreage crops if in add list
if crop_num in crop_add_list:
pass
# Skip and/or remove zones without crop acreage
elif crop_acreage_dict[crop_num][cell_id] < area_threshold:
if remove_empty_flag:
input_lyr.DeleteFeature(input_ftr.GetFID())
continue
# Write parameter values
for param_field, param_method, param_type in param_list:
input_ftr.SetField(input_ftr.GetFieldIndex(param_field),
getattr(crop_param, param_method))
# Write crop acreage
if crop_num not in crop_add_list:
input_ftr.SetField(input_ftr.GetFieldIndex(crop_acres_field),
crop_acreage_dict[crop_num][cell_id])
input_lyr.SetFeature(input_ftr)
input_ds = None
def main(ini_path, zone_type='huc8', area_threshold=10,
dairy_cuttings=5, beef_cuttings=4, crop_str='',
remove_empty_flag=True, overwrite_flag=False):
"""Build a feature class for each crop and set default crop parameters
Apply the values in the CropParams.txt as defaults to every cell
Args:
ini_path (str): file path of the project INI file
zone_type (str): Zone type (huc8, huc10, county, gridmet)
area_threshold (float): CDL area threshold [acres]
dairy_cuttings (int): Initial number of dairy hay cuttings
beef_cuttings (int): Initial number of beef hay cuttings
crop_str (str): comma separated list or range of crops to compare
overwrite_flag (bool): If True, overwrite existing output rasters
Returns:
None
"""
logging.info('\nCalculating ET-Demands Spatial Crop Parameters')
remove_empty_flag = True
# Input paths
# DEADBEEF - For now, get cropET folder from INI file
# This function may eventually be moved into the main cropET code
crop_et_sec = 'CROP_ET'
config = util.read_ini(ini_path, section=crop_et_sec)
try:
project_ws = config.get(crop_et_sec, 'project_folder')
except:
logging.error('project_folder parameter must be set in the INI file, '
'exiting')
return False
try:
gis_ws = config.get(crop_et_sec, 'gis_folder')
except:
logging.error('gis_folder parameter must be set in the INI file, '
'exiting')
return False
try:
cells_path = config.get(crop_et_sec, 'cells_path')
except:
# cells_path = os.path.join(gis_ws, 'ETCells.shp')
logging.error('et_cells_path parameter must be set in the INI file, '
'exiting')
return False
try:
stations_path = config.get(crop_et_sec, 'stations_path')
except:
logging.error('stations_path parameter must be set in the INI file, '
'exiting')
return False
crop_et_ws = config.get(crop_et_sec, 'crop_et_folder')
bin_ws = os.path.join(crop_et_ws, 'bin')
try:
calibration_ws = config.get(crop_et_sec, 'spatial_cal_folder')
except:
calibration_ws = os.path.join(project_ws, 'calibration')
# Sub folder names
static_ws = os.path.join(project_ws, 'static')
# pmdata_ws = os.path.join(project_ws, 'pmdata')
crop_params_path = os.path.join(static_ws, 'CropParams.txt')
# ET cells field names
cell_id_field = 'CELL_ID'
cell_name_field = 'CELL_NAME'
crop_acres_field = 'CROP_ACRES'
# Only keep the following ET Cell fields
keep_field_list = [cell_id_field, cell_name_field, 'AG_ACRES']
# keep_field_list = ['CELL_ID', 'STATION_ID', 'HUC8', 'HUC10', 'GRIDMET_ID,
# 'COUNTYNAME', 'AG_ACRES']
# keep_field_list = ['FIPS', 'COUNTYNAME']
# Check input folders
if not os.path.isdir(crop_et_ws):
logging.error('\nERROR: The INI cropET folder does not exist'
'\n {}'.format(crop_et_ws))
sys.exit()
elif not os.path.isdir(bin_ws):
logging.error('\nERROR: The bin workspace does not exist'
'\n {}'.format(bin_ws))
sys.exit()
elif not os.path.isdir(project_ws):
logging.error('\nERROR: The project folder does not exist'
'\n {}'.format(project_ws))
sys.exit()
elif not os.path.isdir(gis_ws):
logging.error('\nERROR: The GIS folder does not exist'
'\n {}'.format(gis_ws))
sys.exit()
if '.gdb' not in calibration_ws and not os.path.isdir(calibration_ws):
os.makedirs(calibration_ws)
logging.info('\nGIS Workspace: {}'.format(gis_ws))
logging.info('Project Workspace: {}'.format(project_ws))
logging.info('CropET Workspace: {}'.format(crop_et_ws))
logging.info('Bin Workspace: {}'.format(bin_ws))
logging.info('Calib. Workspace: {}'.format(calibration_ws))
# Check input files
if not os.path.isfile(crop_params_path):
logging.error('\nERROR: The crop parameters file does not exist'
'\n {}'.format(crop_params_path))
sys.exit()
elif not os.path.isfile(cells_path):
logging.error('\nERROR: The ET Cell shapefile does not exist'
'\n {}'.format(cells_path))
sys.exit()
elif not os.path.isfile(stations_path) or not arcpy.Exists(stations_path):
logging.error('\nERROR: The weather station shapefile does not exist'
'\n {}'.format(stations_path))
sys.exit()
logging.debug('Crop Params Path: {}'.format(crop_params_path))
logging.debug('ET Cells Path: {}'.format(cells_path))
logging.debug('Stations Path: {}'.format(stations_path))
# For now, only allow calibration parameters in separate shapefiles
ext = '.shp'
# # Build output geodatabase if necessary
# if calibration_ws.endswith('.gdb'):
# logging.debug('GDB Path: {}'.format(calibration_ws))
# ext = ''
# if arcpy.Exists(calibration_ws) and overwrite_flag:
# try: arcpy.Delete_management(calibration_ws)
# except: pass
# if calibration_ws is not None and not arcpy.Exists(calibration_ws):
# arcpy.CreateFileGDB_management(
# os.path.dirname(calibration_ws),
# os.path.basename(calibration_ws))
# else:
# ext = '.shp'
# Field Name, Property, Field Type
# Property is the string of the CropParameter class property value
# It will be used to access the property using getattr
dairy_cutting_field = 'Dairy_Cut'
beef_cutting_field = 'Beef_Cut'
param_list = [
# ['Name', 'name', 'STRING'],
# ['ClassNum', 'class_number', 'LONG'],
# ['IsAnnual', 'is_annual', 'SHORT'],
# ['IrrigFlag', 'irrigation_flag', 'SHORT'],
# ['IrrigDays', 'days_after_planting_irrigation', 'LONG'],
# ['Crop_FW', 'crop_fw', 'LONG'],
# ['WinterCov', 'winter_surface_cover_class', 'SHORT'],
# ['CropKcMax', 'kc_max', 'FLOAT'],
['MAD_Init', 'mad_initial', 'LONG'],
['MAD_Mid', 'mad_midseason', 'LONG'],
# ['RootDepIni', 'rooting_depth_initial', 'FLOAT'],
# ['RootDepMax', 'rooting_depth_max', 'FLOAT'],
# ['EndRootGrw', 'end_of_root_growth_fraction_time', 'FLOAT'],
# ['HeightInit', 'height_initial', 'FLOAT'],
# ['HeightMax', 'height_max', 'FLOAT'],
# ['CurveNum', 'curve_number', 'LONG'],
# ['CurveName', 'curve_name', 'STRING'],
# ['CurveType', 'curve_type', 'SHORT'],
# ['PL_GU_Flag', 'flag_for_means_to_estimate_pl_or_gu', 'SHORT'],
['T30_CGDD', 't30_for_pl_or_gu_or_cgdd', 'FLOAT'],
['PL_GU_Date', 'date_of_pl_or_gu', 'FLOAT'],
['CGDD_Tbase', 'tbase', 'FLOAT'],
['CGDD_EFC', 'cgdd_for_efc', 'LONG'],
['CGDD_Term', 'cgdd_for_termination', 'LONG'],
['Time_EFC', 'time_for_efc', 'LONG'],
['Time_Harv', 'time_for_harvest', 'LONG'],
['KillFrostC', 'killing_frost_temperature', 'FLOAT'],
# ['InvokeStrs', 'invoke_stress', 'SHORT'],
# ['CN_Coarse', 'cn_coarse_soil', 'LONG'],
# ['CN_Medium', 'cn_medium_soil', 'LONG'],
# ['CN_Fine', 'cn_fine_soil', 'LONG']
]
# if calibration_ws.endswith('.gdb'):
# dairy_cutting_field = 'Dairy_Cuttings'
# beef_cutting_field = 'Beef_Cuttings'
# param_list = [
# # ['Name', 'name', 'STRING'],
# # ['Class_Number', 'class_number', 'LONG'],
# # ['Is_Annual', 'is_annual', 'SHORT'],
# # ['Irrigation_Flag', 'irrigation_flag', 'SHORT'],
# # ['Irrigation_Days', 'days_after_planting_irrigation', 'LONG'],
# # ['Crop_FW', 'crop_fw', 'LONG'],
# # ['Winter_Cover_Class', 'winter_surface_cover_class', 'SHORT'],
# # ['Crop_Kc_Max', 'kc_max', 'FLOAT'],
# # ['MAD_Initial', 'mad_initial', 'LONG'],
# # ['MAD_Midseason', 'mad_midseason', 'LONG'],
# # ['Root_Depth_Ini', 'rooting_depth_initial', 'FLOAT'],
# # ['Root_Depth_Max', 'rooting_depth_max', 'FLOAT'],
# # ['End_Root_Growth', 'end_of_root_growth_fraction_time', 'FLOAT'],
# # ['Height_Initial', 'height_initial', 'FLOAT'],
# # ['Height_Maximum', 'height_max', 'FLOAT'],
# # ['Curve_Number', 'curve_number', 'LONG'],
# # ['Curve_Name', 'curve_name', 'STRING'],
# # ['Curve_Type', 'curve_type', 'SHORT'],
# # ['PL_GU_Flag', 'flag_for_means_to_estimate_pl_or_gu', 'SHORT'],
# ['T30_CGDD', 't30_for_pl_or_gu_or_cgdd', 'FLOAT'],
# ['PL_GU_Date', 'date_of_pl_or_gu', 'FLOAT'],
# ['CGDD_Tbase', 'tbase', 'FLOAT'],
# ['CGDD_EFC', 'cgdd_for_efc', 'LONG'],
# ['CGDD_Termination', 'cgdd_for_termination', 'LONG'],
# ['Time_EFC', 'time_for_efc', 'LONG'],
# ['Time_Harvest', 'time_for_harvest', 'LONG'],
# ['Killing_Crost_C', 'killing_frost_temperature', 'FLOAT'],
# # ['Invoke_Stress', 'invoke_stress', 'SHORT'],
# # ['CN_Coarse_Soil', 'cn_coarse_soil', 'LONG'],
# # ['CN_Medium_Soil', 'cn_medium_soil', 'LONG'],
# # ['CN_Fine_Soil', 'cn_fine_soil', 'LONG']
# ]
crop_add_list = []
if crop_str:
try:
crop_add_list = sorted(list(util.parse_int_set(crop_str)))
# try:
# crop_test_list = sorted(list(set(
# crop_test_list + list(util.parse_int_set(crop_str)))
except:
pass
# Don't build crop parameter files for non-crops
crop_skip_list = sorted(list(set([44, 45, 46, 55, 56, 57])))
# crop_test_list = sorted(list(set(crop_test_list + [46])))
logging.info('\ncrop_add_list = {}'.format(crop_add_list))
# Read crop parameters using ET Demands functions/methods
logging.info('\nReading default crop parameters')
sys.path.append(bin_ws)
import crop_parameters
crop_param_dict = crop_parameters.read_crop_parameters(crop_params_path)
# arcpy.CheckOutExtension('Spatial')
# arcpy.env.pyramid = 'NONE 0'
arcpy.env.overwriteOutput = overwrite_flag
arcpy.env.parallelProcessingFactor = 8
# Get list of crops specified in ET cells
# Currently this may only be crops with CDL acreage
crop_field_list = [
field.name for field in arcpy.ListFields(cells_path)
if re.match('CROP_\d{2}', field.name)]
logging.debug('Cell crop fields: {}'.format(', '.join(crop_field_list)))
crop_number_list = [
int(f_name.split('_')[-1]) for f_name in crop_field_list]
crop_number_list = [
crop_num for crop_num in crop_number_list
if not (crop_skip_list and crop_num in crop_skip_list)]
logging.info('Cell crop numbers: {}'.format(
', '.join(list(util.ranges(crop_number_list)))))
# Get crop acreages for each cell
crop_acreage_dict = defaultdict(dict)
field_list = [cell_id_field] + crop_field_list
with arcpy.da.SearchCursor(cells_path, field_list) as cursor:
for row in cursor:
for i, crop_num in enumerate(crop_number_list):
# logging.info('{} {}'.format(crop_num, i))
if crop_num in crop_add_list:
crop_acreage_dict[crop_num][row[0]] = 0
else:
crop_acreage_dict[crop_num][row[0]] = row[i + 1]
crop_number_list = sorted(list(set(crop_number_list) | set(crop_add_list)))
# Make an empty template crop feature class
logging.info('')
crop_template_path = os.path.join(
calibration_ws, 'crop_00_template' + ext)
if overwrite_flag and arcpy.Exists(crop_template_path):
logging.debug('Overwriting template crop feature class')
arcpy.Delete_management(crop_template_path)
if arcpy.Exists(crop_template_path):
logging.info('Template crop feature class already exists, skipping')
else:
logging.info('Building template crop feature class')
arcpy.CopyFeatures_management(cells_path, crop_template_path)
# Remove unneeded et cell fields
for field in arcpy.ListFields(crop_template_path):
if (field.name not in keep_field_list and
field.editable and not field.required):
logging.debug(' Delete field: {}'.format(field.name))
arcpy.DeleteField_management(crop_template_path, field.name)
field_list = [f.name for f in arcpy.ListFields(crop_template_path)]
# Add crop acreage field
if crop_acres_field not in field_list:
logging.debug(' Add field: {}'.format(crop_acres_field))
arcpy.AddField_management(
crop_template_path, crop_acres_field, 'Float')
arcpy.CalculateField_management(
crop_template_path, crop_acres_field, '0', 'PYTHON_9.3')
# Add crop parameter fields if necessary
for param_field, param_method, param_type in param_list:
logging.debug(' Add field: {}'.format(param_field))
if param_field not in field_list:
arcpy.AddField_management(
crop_template_path, param_field, param_type)
# if dairy_cutting_field not in field_list:
# logging.debug(' Add field: {}'.format(dairy_cutting_field))
# arcpy.AddField_management(crop_template_path, dairy_cutting_field, 'Short')
# arcpy.CalculateField_management(
# crop_template_path, dairy_cutting_field, dairy_cuttings, 'PYTHON')
# if beef_cutting_field not in field_list:
# logging.debug(' Add field: {}'.format(beef_cutting_field))
# arcpy.AddField_management(crop_template_path, beef_cutting_field, 'Short')
# arcpy.CalculateField_management(
# crop_template_path, beef_cutting_field, beef_cuttings, 'PYTHON')
# Add an empty/zero crop field for the field mappings below
# if len(arcpy.ListFields(cells_path, 'CROP_EMPTY')) == 0:
# arcpy.AddField_management(cells_path, 'CROP_EMPTY', 'Float')
# arcpy.CalculateField_management(
# cells_path, 'CROP_EMPTY', '0', 'PYTHON_9.3')
# Process each crop
logging.info('\nBuilding crop feature classes')
for crop_num in crop_number_list:
try:
crop_param = crop_param_dict[crop_num]
except:
continue
logging.info('{:>2d} {}'.format(crop_num, crop_param.name))
logging.debug('{}'.format(crop_param))
# Replace other characters with spaces, then remove multiple spaces
crop_name = re.sub('[-"().,/~]', ' ', str(crop_param.name).lower())
crop_name = ' '.join(crop_name.strip().split()).replace(' ', '_')
crop_path = os.path.join(calibration_ws, 'crop_{0:02d}_{1}{2}'.format(
crop_num, crop_name, ext))
# crop_field = 'CROP_{:02d}'.format(crop_num)
# Don't check crops in add list
if crop_num in crop_add_list:
pass
# Skip if all zone crop areas are below threshold
elif all([v < area_threshold for v in
crop_acreage_dict[crop_num].values()]):
logging.info(' All crop acreaeges below threshold, skipping crop')
continue
# Remove existing shapefiles if necessary
if overwrite_flag and arcpy.Exists(crop_path):
logging.debug(' Overwriting: {}'.format(
os.path.basename(crop_path)))
arcpy.Delete_management(crop_path)
# Don't check skip list until after existing files are removed
# if ((crop_test_list and crop_num not in crop_test_list) or
# _skip_list and crop_num in crop_skip_list)):
# .debug(' Skipping')
# Copy ET cells for each crop if needed
if arcpy.Exists(crop_path):
logging.debug(' Shapefile already exists, skipping')
continue
else:
# logging.debug(' {}'.format(crop_path))
arcpy.Copy_management(crop_template_path, crop_path)
# Remove extra fields
# for field in arcpy.ListFields(crop_path):
# if field.name not in keep_field_list:
# # logging.debug(' {}'.format(field.name))
# arcpy.DeleteField_management(crop_path, field.name)
# Add alfalfa cutting field
if crop_num in [1, 2, 3, 4]:
if len(arcpy.ListFields(crop_path, dairy_cutting_field)) == 0:
logging.debug(' Add field: {}'.format(dairy_cutting_field))
arcpy.AddField_management(
crop_path, dairy_cutting_field, 'Short')
arcpy.CalculateField_management(
crop_path, dairy_cutting_field, dairy_cuttings, 'PYTHON')
if len(arcpy.ListFields(crop_path, beef_cutting_field)) == 0:
logging.debug(' Add field: {}'.format(beef_cutting_field))
arcpy.AddField_management(
crop_path, beef_cutting_field, 'Short')
arcpy.CalculateField_management(
crop_path, beef_cutting_field, beef_cuttings, 'PYTHON')
# Write default crop parameters to file
field_list = [p[0] for p in param_list] + [cell_id_field, crop_acres_field]
with arcpy.da.UpdateCursor(crop_path, field_list) as cursor:
for row in cursor:
# Don't remove zero acreage crops if in add list
if crop_num in crop_add_list:
pass
# Skip and/or remove zones without crop acreage
elif crop_acreage_dict[crop_num][row[-2]] < area_threshold:
if remove_empty_flag:
cursor.deleteRow()
continue
# Write parameter values
for i, (param_field, param_method, param_type) in enumerate(param_list):
row[i] = getattr(crop_param, param_method)
# Write crop acreage
if crop_num not in crop_add_list:
row[-1] = crop_acreage_dict[crop_num][row[-2]]
cursor.updateRow(row)
def main(gis_ws,
cdl_ws,
cdl_year,
study_area_path,
study_area_buffer=None,
overwrite_flag=False,
pyramids_flag=False,
stats_flag=False):
"""Build study area raster from a target extent and rebuild color table
Args:
gis_ws (str): Folder/workspace path of the GIS data for the project
cdl_ws (str): Folder/workspace path of the GIS data for the project
cdl_year (str): Cropland Data Layer year
zone_path (str): File path to study area shapefile
zone_buffer (float): Distance to buffer input extent
Units will be the same as the extent spatial reference
overwrite_flag (bool): If True, overwrite output raster
pyramids_flag (bool): If True, build pyramids/overviews
for the output raster
stats_flag (bool): If True, compute statistics for the output raster
Returns:
None
"""
scratch_ws = os.path.join(gis_ws, 'scratch')
zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img')
zone_polygon_path = os.path.join(scratch_ws, 'zone_polygon.shp')
# If multiple years were passed in, only use the first one
cdl_year = list(util.parse_int_set(cdl_year))[0]
cdl_format = '{}_30m_cdls.img'
cdl_path = os.path.join(cdl_ws, cdl_format.format(cdl_year))
# Reference all output rasters to CDL
# output_osr = gdc.raster_path_osr(cdl_path)
output_proj = gdc.raster_path_proj(cdl_path)
output_cs = gdc.raster_path_cellsize(cdl_path)[0]
output_x, output_y = gdc.raster_path_origin(cdl_path)
# output_osr = gdc.proj4_osr(
# "+proj=aea +lat_1=29.5 +lat_2=45.5 +lat_0=23 +lon_0=-96 "+
# "+x_0=0 +y_0=0 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m "+
# "+no_defs")
# output_cs = 30
# output_x, output_y = 15, 15
output_format = 'HFA'
if pyramids_flag:
levels = '2 4 8 16 32 64 128'
# gdal.SetConfigOption('USE_RRD', 'YES')
# gdal.SetConfigOption('HFA_USE_RRD', 'YES')
# gdal.SetConfigOption('HFA_COMPRESS_OVR', 'YES')
if os.name == 'posix':
shell_flag = False
else:
shell_flag = True
# Check input folders
if not os.path.isdir(gis_ws):
logging.error('\nERROR: The GIS workspace does not exist'
'\n {}'.format(gis_ws))
sys.exit()
elif not os.path.isfile(cdl_path):
logging.error('\nERROR: The input CDL raster does not exist'
'\n {}'.format(cdl_path))
sys.exit()
elif not os.path.isfile(study_area_path):
logging.error('\nERROR: The extent shapefile does not exist'
'\n {}'.format(study_area_path))
sys.exit()
if not os.path.isdir(scratch_ws):
os.makedirs(scratch_ws)
logging.info('\nGIS Workspace: {}'.format(gis_ws))
logging.info('Scratch Workspace: {}'.format(scratch_ws))
# Overwrite
if os.path.isfile(zone_raster_path) and overwrite_flag:
subprocess.check_output(
['gdalmanage', 'delete', '-f', output_format, zone_raster_path],
shell=shell_flag)
if os.path.isfile(zone_polygon_path) and overwrite_flag:
remove_file(zone_polygon_path)
# subprocess.check_output(
# ['gdalmanage', 'delete', zone_polygon_path],
# shell=shell_flag)
# Project extent shapefile to CDL spatial reference
if not os.path.isfile(zone_polygon_path):
# Project study area extent to the input/CDL spatial reference
logging.info('Projecting extent shapefile')
subprocess.check_output([
'ogr2ogr', '-f', 'ESRI Shapefile', '-overwrite', '-preserve_fid',
'-unsetFieldWidth', '-t_srs',
str(output_proj), zone_polygon_path, study_area_path
],
shell=shell_flag)
# Get the study area extent from the projected shapefile
clip_extent = gdc.feature_path_extent(zone_polygon_path)
logging.debug('Clip Extent: {}'.format(clip_extent))
# This will buffer in the CDL spatial reference & units
if study_area_buffer is not None:
logging.debug('Buffering: {}'.format(study_area_buffer))
clip_extent.buffer(study_area_buffer)
logging.debug('Clip Extent: {}'.format(clip_extent))
clip_extent.adjust_to_snap(output_x, output_y, output_cs, method='EXPAND')
logging.debug('Clip Extent: {}'.format(clip_extent))
# gdal_translate uses ul/lr corners, not extent
clip_ullr = clip_extent.ul_lr_swap()
logging.debug('Clip UL/LR: {}'.format(clip_ullr))
# Rasterize extent shapefile for masking in other scripts
if (not os.path.isfile(zone_raster_path)
and os.path.isfile(zone_polygon_path)):
logging.info('Rasterizing shapefile')
subprocess.check_output([
'gdal_rasterize', '-of', output_format, '-ot', 'Byte', '-burn',
'1', '-init', '0', '-a_nodata', '255', '-co', 'COMPRESSED=YES'
] + ['-te'] + str(clip_extent).split() + [
'-tr',
str(output_cs),
str(output_cs), zone_polygon_path, zone_raster_path
],
shell=shell_flag)
# remove_file(zonse_polygon_path)
# Statistics
if stats_flag and os.path.isfile(zone_raster_path):
logging.info('Computing statistics')
logging.debug(' {}'.format(zone_raster_path))
subprocess.check_output([
'gdalinfo', '-stats', '-nomd', '-noct', '-norat', zone_raster_path
],
shell=shell_flag)
# Pyramids
if pyramids_flag and os.path.isfile(zone_raster_path):
logging.info('Building statistics')
logging.debug(' {}'.format(zone_raster_path))
subprocess.check_output(['gdaladdo', '-ro', zone_raster_path] +
levels.split(),
shell=shell_flag)
def main(gis_ws, cdl_year='', block_size=16384, mask_flag=False,
overwrite_flag=False, pyramids_flag=False, stats_flag=False,
agland_nodata=0, agmask_nodata=0):
"""Mask CDL values for non-agricultural pixels
Use CDL derived agmask (in CDL workspace) to define agricultural pixels
Args:
gis_ws (str): Folder/workspace path of the GIS data for the project
cdl_year (str): Comma separated list and/or range of years
block_size (int): Maximum block size to use for raster processing
mask_flag (bool): If True, mask pixels outside extent shapefile
overwrite_flag (bool): If True, overwrite output rasters
pyramids_flag (bool): If True, build pyramids/overviews
for the output rasters
stats_flag (bool): If True, compute statistics for the output rasters
agland_nodata: Integer of the nodata value in the agland raster
agmask_nodata: Integer of the nodata value in the agmask raster
Returns:
None
"""
logging.info('\nExtracting Agriculatural CDL Values')
cdl_format = '{0}_30m_cdls.img'
cdl_ws = os.path.join(gis_ws, 'cdl')
scratch_ws = os.path.join(gis_ws, 'scratch')
zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img')
output_format = 'HFA'
# Ag landuses are 1, all others in state are 0, outside state is nodata
# Crop 61 is fallow/idle and was excluded from analysis
# Crop 176 is Grassland/Pasture in the new national CDL rasters
# Crop 181 was Pasture/Hay in the old state CDL rasters
# Crop 182 was Cultivated Crop in the old state CDL rasters
agmask_remap = [
[1, 60, 1], [61, 65, 0],
# [1, 61, 1], [62, 65, 0],
[66, 80, 1], [81, 92, 0],
# [93, 180, 0], [181, 182, 1], [183, 203, 0], [204, 254, 1]]
# [93, 180, 0], [176, 1], [183, 203, 0], [204, 254, 1]]
[97, 100, 1], [101, 203, 0], [204, 254, 1]]
if pyramids_flag:
levels = '2 4 8 16 32 64 128'
# gdal.SetConfigOption('USE_RRD', 'YES')
# gdal.SetConfigOption('HFA_USE_RRD', 'YES')
# gdal.SetConfigOption('HFA_COMPRESS_OVR', 'YES')
if os.name == 'posix':
shell_flag = False
else:
shell_flag = True
# Check input folders
if not os.path.isdir(gis_ws):
logging.error('\nERROR: The GIS workspace does not exist'
'\n {}'.format(gis_ws))
sys.exit()
elif not os.path.isdir(cdl_ws):
logging.error('\nERROR: The CDL workspace does not exist'
'\n {}'.format(cdl_ws))
sys.exit()
elif mask_flag and not os.path.isfile(zone_raster_path):
logging.error(
'\nERROR: The zone raster {} does not exist\n'
' Try re-running "clip_cdl_raster.py"'.format(zone_raster_path))
sys.exit()
logging.info('\nGIS Workspace: {}'.format(gis_ws))
logging.info('CDL Workspace: {}'.format(cdl_ws))
# Process each CDL year separately
for cdl_year in list(util.parse_int_set(cdl_year)):
logging.info('\n{}'.format(cdl_year))
cdl_path = os.path.join(cdl_ws, cdl_format.format(cdl_year))
agmask_path = os.path.join(
cdl_ws, 'agmask_{}_30m_cdls.img'.format(cdl_year))
agland_path = os.path.join(
cdl_ws, 'agland_{}_30m_cdls.img'.format(cdl_year))
if not os.path.isfile(cdl_path):
logging.error('\nERROR: The CDL raster does not exist'
'\n {}'.format(cdl_path))
continue
# Get color table and spatial reference from CDL raster
logging.info('Reading CDL color table')
cdl_raster_ds = gdal.Open(cdl_path, 0)
cdl_geo = gdc.raster_ds_geo(cdl_raster_ds)
cdl_rows, cdl_cols = gdc.raster_ds_shape(cdl_raster_ds)
cdl_extent = gdc.geo_extent(cdl_geo, cdl_rows, cdl_cols)
cdl_proj = gdc.raster_ds_proj(cdl_raster_ds)
# DEADBEEF - Why is this hardcoded?
# cdl_cellsize = 30
cdl_cellsize = gdc.raster_ds_cellsize(cdl_raster_ds)[0]
# cdl_band = cdl_raster_ds.GetRasterBand(1)
# cdl_rat = cdl_band.GetDefaultRAT()
# cdl_classname_dict = dict()
# for row_i in range(cdl_rat.GetRowCount()):
# cdl_classname_dict[row_i] = cdl_rat.GetValueAsString(
# row_i, cdl_rat.GetColOfUsage(2))
# cdl_raster_ds = None
# del cdl_raster_ds, cdl_band, cdl_rat
# Copy the input raster to hold the ag data
logging.debug('{}'.format(agland_path))
if os.path.isfile(agland_path) and overwrite_flag:
subprocess.check_output(
['gdalmanage', 'delete', '-f', output_format, agland_path],
shell=shell_flag)
if not os.path.isfile(agland_path):
logging.info('Copying CDL raster')
logging.debug('{}'.format(cdl_path))
subprocess.check_output(
['gdal_translate', '-of', output_format, '-co', 'COMPRESSED=YES',
cdl_path, agland_path],
shell=shell_flag)
# '-a_nodata', agland_nodata
if os.path.isfile(cdl_path.replace('.img', '.img.vat.dbf')):
shutil.copyfile(
cdl_path.replace('.img', '.img.vat.dbf'),
agland_path.replace('.img', '.img.vat.dbf')
)
# Set the nodata value after copying
agland_ds = gdal.Open(agland_path, 1)
agland_band = agland_ds.GetRasterBand(1)
agland_band.SetNoDataValue(agland_nodata)
agland_ds = None
# # Get the colormap from the input CDL raster
# logging.debug('Re-building raster attribute tables')
# agland_ds = gdal.Open(agland_path, 1)
# agland_band = agland_ds.GetRasterBand(1)
# agland_rat = agland_band.GetDefaultRAT()
# agland_usage_list = [
# agland_rat.GetUsageOfCol(col_i)
# for col_i in range(agland_rat.GetColumnCount())]
# # if 1 not in agland_usage_list:
# # _rat.CreateColumn('Count', 1, 1)
# # WHY IS THE LOOKING AT Class_Name??? -CHRIS
# if 2 not in agland_usage_list:
# agland_rat.CreateColumn('Class_Name', 2, 2)
# for row_i in range(agland_rat.GetRowCount()):
# agland_rat.SetValueAsString(
# row_i, agland_rat.GetColOfUsage(2),
# cdl_classname_dict[row_i])
# agland_band.SetDefaultRAT(agland_rat)
# agland_ds = None
# Build an empty output raster to hold the ag mask
logging.info('\nBuilding empty ag mask raster')
logging.debug('{}'.format(agmask_path))
if os.path.isfile(agmask_path) and overwrite_flag:
subprocess.check_output(
['gdalmanage', 'delete', '-f', output_format, agmask_path],
shell=shell_flag)
if not os.path.isfile(agmask_path):
gdc.build_empty_raster(
agmask_path, band_cnt=1, output_dtype=np.uint8,
output_nodata=agmask_nodata, output_proj=cdl_proj,
output_cs=cdl_cellsize, output_extent=cdl_extent)
# Set the nodata value after initializing
agmask_ds = gdal.Open(agmask_path, 1)
agmask_band = agmask_ds.GetRasterBand(1)
agmask_band.SetNoDataValue(agmask_nodata)
agmask_ds = None
# Set non-ag areas to nodata value
logging.info('\nProcessing by block')
logging.debug(' Input cols/rows: {0}/{1}'.format(cdl_cols, cdl_rows))
for b_i, b_j in gdc.block_gen(cdl_rows, cdl_cols, block_size):
logging.info(' Block y: {0:5d} x: {1:5d}'.format(b_i, b_j))
# Read in data for block
cdl_array = gdc.raster_to_block(
cdl_path, b_i, b_j, block_size,
fill_value=0, return_nodata=False)
cdl_mask = np.zeros(cdl_array.shape, dtype=np.bool)
remap_mask = np.zeros(cdl_array.shape, dtype=np.bool)
# Mask CDL values outside extent shapefile
if mask_flag and os.path.isfile(zone_raster_path):
zone_array = gdc.raster_to_block(
zone_raster_path, b_i, b_j, block_size)
cdl_array[zone_array == 0] = 0
# Reclassify to 1 for ag and 0 for non-ag
for [start, end, value] in agmask_remap:
if value == 0:
continue
logging.debug([start, end, value])
remap_mask |= (cdl_array >= start) & (cdl_array <= end)
cdl_mask[remap_mask] = True
del remap_mask
# Set non-ag areas in agmask to nodata
cdl_mask[~cdl_mask] = agmask_nodata
# Set non-ag areas in aglands to nodata
cdl_array[~cdl_mask] = agland_nodata
gdc.block_to_raster(cdl_array, agland_path, b_i, b_j, block_size)
gdc.block_to_raster(cdl_mask.astype(np.uint8), agmask_path,
b_i, b_j, block_size)
del cdl_array, cdl_mask
if stats_flag:
logging.info('Computing statistics')
if os.path.isfile(agland_path):
logging.debug('{}'.format(agland_path))
subprocess.check_output(
['gdalinfo', '-stats', '-nomd', '-noct', '-norat',
agland_path], shell=shell_flag)
if os.path.isfile(agmask_path):
logging.debug('{}'.format(agmask_path))
subprocess.check_output(
['gdalinfo', '-stats', '-nomd', '-noct', '-norat',
agmask_path], shell=shell_flag)
if pyramids_flag:
logging.info('Building pyramids')
if os.path.isfile(agland_path):
logging.debug('{}'.format(agland_path))
subprocess.check_output(
['gdaladdo', '-ro', agland_path] + levels.split(),
shell=shell_flag)
# args = ['gdaladdo', '-ro']
# if agland_path.endswith('.img'):
# args.extend([
# '--config', 'USE_RRD YES',
# '--config', 'HFA_USE_RRD YES',
# '--config', 'HFA_COMPRESS_OVR YES'])
# args.append(agland_path)
# args.extend(levels.split())
# subprocess.check_output(args, shell=shell_flag)
if os.path.isfile(agmask_path):
logging.debug('{}'.format(agmask_path))
subprocess.check_output(
['gdaladdo', '-ro', agmask_path] + levels.split(),
shell=shell_flag)