def gcs_to_utm_dict(tile_list,
tile_utm_zone_dict,
tile_gcs_osr,
tile_gcs_wkt_dict,
gcs_buffer=0.25,
snap_xmin=None,
snap_ymin=None,
snap_cs=None):
"""Return a dictionary of Landsat path/row GCS extents projected to UTM
Parameters
----------
tile_list : list
tile_utm_zone_dict : dict
tile_gcs_osr :
tile_gcs_wkt_dict :
gcs_buffer : float, optional
snap_xmin : float or None, optional
snap_ymin : float or None, optional
snap_cs : float or None, optional
Returns
-------
dict
"""
# If parameters are not set, try to get from env
# if snap_xmin is None and env.snap_xmin:
# snap_xmin = env.snap_xmin
# if snap_ymin is None and env.snap_ymin:
# snap_ymin = env.snap_ymin
# if snap_cs is None and env.cellsize:
# snap_cs = env.cellsize
logging.info('\nCalculate projected extent for each path/row')
output_dict = dict()
for tile_name in sorted(tile_list):
logging.info(' {}'.format(tile_name))
# Create an OSR object from the utm projection
tile_utm_osr = drigo.epsg_osr(32600 +
int(tile_utm_zone_dict[tile_name]))
# tile_utm_proj = drigo.osr_proj(tile_utm_osr)
# Create utm transformation
tile_utm_tx = osr.CoordinateTransformation(tile_gcs_osr, tile_utm_osr)
tile_gcs_geom = ogr.CreateGeometryFromWkt(tile_gcs_wkt_dict[tile_name])
# Buffer extent by 0.1 degrees
# DEADBEEF - Buffer fails if GDAL is not built with GEOS support
# tile_gcs_geom = tile_gcs_geom.Buffer(gcs_buffer)
# Create gcs to utm transformer and apply it
tile_utm_geom = tile_gcs_geom.Clone()
tile_utm_geom.Transform(tile_utm_tx)
tile_utm_extent = drigo.Extent(tile_utm_geom.GetEnvelope())
tile_utm_extent = tile_utm_extent.ogrenv_swap()
# 0.1 degrees ~ 10 km
tile_utm_extent.buffer_extent(gcs_buffer * 100000)
tile_utm_extent.adjust_to_snap('EXPAND', snap_xmin, snap_ymin, snap_cs)
output_dict[tile_name] = tile_utm_extent
return output_dict
def unknown_proj_osr(input_proj):
"""Return the spatial reference object for a projection string"""
try:
output_osr = drigo.epsg_osr(input_proj)
logging.debug(' OSR from EPSG string')
return output_osr
except:
pass
try:
output_osr = drigo.epsg_osr(input_proj.replace('EPSG:'))
logging.debug(' OSR from EPSG integer')
return output_osr
except:
pass
try:
output_osr = drigo.proj_osr(input_proj)
logging.debug(' OSR from WKT')
return output_osr
except:
pass
try:
output_osr = drigo.proj4_osr(input_proj)
logging.debug(' OSR from PROJ4')
return output_osr
except:
pass
try:
output_osr = drigo.raster_path_osr(input_proj)
logging.debug(' OSR from raster path')
return output_osr
except:
pass
try:
output_osr = drigo.feature_path_osr(input_proj)
logging.debug(' OSR from feature path')
return output_osr
except:
pass
return output_osr
def main(netcdf_ws=os.getcwd(), ancillary_ws=os.getcwd(),
output_ws=os.getcwd(), variables=['prcp'],
daily_flag=False, monthly_flag=True, annual_flag=False,
start_year=1981, end_year=2010,
extent_path=None, output_extent=None,
stats_flag=True, overwrite_flag=False):
"""Extract DAYMET temperature
Parameters
----------
netcdf_ws : str
Folder of DAYMET netcdf files.
ancillary_ws : str
Folder of ancillary rasters.
output_ws : str
Folder of output rasters.
variables : list, optional
DAYMET variables to download ('prcp', 'srad', 'vp', 'tmmn', 'tmmx').
Set as ['all'] to process all variables.
daily_flag : bool, optional
If True, compute daily (DOY) climatologies.
monthly_flag : bool, optional
If True, compute monthly climatologies.
annual_flag : bool, optional
If True, compute annual climatologies.
start_year : int, optional
Climatology start year.
end_year : int, optional
Climatology end year.
extent_path : str, optional
File path a raster defining the output extent.
output_extent : list, optional
Decimal degrees values defining output extent.
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nGenerating DAYMET climatologies')
daily_fmt = 'daymet_{var}_30yr_normal_{doy:03d}.img'
monthly_fmt = 'daymet_{var}_30yr_normal_{month:02d}.img'
annual_fmt = 'daymet_{var}_30yr_normal.img'
# daily_fmt = 'daymet_{var}_normal_{start}_{end}_{doy:03d}.img'
# monthly_fmt = 'daymet_{var}_normal_{start}_{end}_{month:02d}.img'
# annual_fmt = 'daymet_{var}_normal_{start}_{end}.img'
# If a date is not set, process 1981-2010 climatology
try:
start_dt = dt.datetime(start_year, 1, 1)
logging.debug(' Start date: {}'.format(start_dt))
except:
start_dt = dt.datetime(1981, 1, 1)
logging.info(' Start date: {}'.format(start_dt))
try:
end_dt = dt.datetime(end_year, 12, 31)
logging.debug(' End date: {}'.format(end_dt))
except:
end_dt = dt.datetime(2010, 12, 31)
logging.info(' End date: {}'.format(end_dt))
# Get DAYMET spatial reference from an ancillary raster
mask_raster = os.path.join(ancillary_ws, 'daymet_mask.img')
daymet_re = re.compile('daymet_v3_(?P<VAR>\w+)_(?P<YEAR>\d{4})_na.nc4$')
# DAYMET rasters to extract
var_full_list = ['prcp', 'tmmn', 'tmmx']
# data_full_list = ['prcp', 'srad', 'vp', 'tmmn', 'tmmx']
if not variables:
logging.error('\nERROR: variables parameter is empty\n')
sys.exit()
elif type(variables) is not list:
# DEADBEEF - I could try converting comma separated strings to lists?
logging.warning('\nERROR: variables parameter must be a list\n')
sys.exit()
elif 'all' in variables:
logging.error('\nDownloading all variables\n {}'.format(
','.join(var_full_list)))
var_list = var_full_list[:]
elif not set(variables).issubset(set(var_full_list)):
logging.error('\nERROR: variables parameter is invalid\n {}'.format(
variables))
sys.exit()
else:
var_list = variables[:]
# Get extent/geo from mask raster
daymet_ds = gdal.Open(mask_raster)
daymet_osr = drigo.raster_ds_osr(daymet_ds)
daymet_proj = drigo.osr_proj(daymet_osr)
daymet_cs = drigo.raster_ds_cellsize(daymet_ds, x_only=True)
daymet_extent = drigo.raster_ds_extent(daymet_ds)
daymet_geo = daymet_extent.geo(daymet_cs)
daymet_x, daymet_y = daymet_extent.origin()
daymet_ds = None
logging.debug(' Projection: {}'.format(daymet_proj))
logging.debug(' Cellsize: {}'.format(daymet_cs))
logging.debug(' Geo: {}'.format(daymet_geo))
logging.debug(' Extent: {}'.format(daymet_extent))
logging.debug(' Origin: {} {}'.format(daymet_x, daymet_y))
# Subset data to a smaller extent
if output_extent is not None:
logging.info('\nComputing subset extent & geo')
logging.debug(' Extent: {}'.format(output_extent))
# Assume input extent is in decimal degrees
output_extent = drigo.project_extent(
drigo.Extent(output_extent), drigo.epsg_osr(4326), daymet_osr, 0.001)
output_extent = drigo.intersect_extents([daymet_extent, output_extent])
output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs)
output_geo = output_extent.geo(daymet_cs)
logging.debug(' Geo: {}'.format(output_geo))
logging.debug(' Extent: {}'.format(output_extent))
elif extent_path is not None:
logging.info('\nComputing subset extent & geo')
output_extent = drigo.project_extent(
drigo.raster_path_extent(extent_path),
drigo.raster_path_osr(extent_path), daymet_osr,
drigo.raster_path_cellsize(extent_path, x_only=True))
output_extent = drigo.intersect_extents([daymet_extent, output_extent])
output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs)
output_geo = output_extent.geo(daymet_cs)
logging.debug(' Geo: {}'.format(output_geo))
logging.debug(' Extent: {}'.format(output_extent))
else:
output_extent = daymet_extent.copy()
output_geo = daymet_geo[:]
output_shape = output_extent.shape(cs=daymet_cs)
xi, yi = drigo.array_geo_offsets(daymet_geo, output_geo, daymet_cs)
output_rows, output_cols = output_extent.shape(daymet_cs)
logging.debug(' Shape: {} {}'.format(output_rows, output_cols))
logging.debug(' Offsets: {} {} (x y)'.format(xi, yi))
# Process each variable
for input_var in var_list:
logging.info("\nVariable: {}".format(input_var))
# Rename variables to match cimis
if input_var == 'prcp':
output_var = 'ppt'
else:
output_var = input_var
logging.debug("Output name: {}".format(output_var))
# Build output folder
var_ws = os.path.join(output_ws, output_var)
if not os.path.isdir(var_ws):
os.makedirs(var_ws)
# Build output arrays
logging.debug(' Building arrays')
if daily_flag:
daily_sum = np.full(
(365, output_shape[0], output_shape[1]), 0, np.float64)
daily_count = np.full(
(365, output_shape[0], output_shape[1]), 0, np.uint8)
if monthly_flag:
monthly_sum = np.full(
(12, output_shape[0], output_shape[1]), 0, np.float64)
monthly_count = np.full(
(12, output_shape[0], output_shape[1]), 0, np.uint8)
if monthly_flag:
annual_sum = np.full(
(output_shape[0], output_shape[1]), 0, np.float64)
annual_count = np.full(
(output_shape[0], output_shape[1]), 0, np.uint8)
# Process each file/year separately
for input_name in sorted(os.listdir(netcdf_ws)):
logging.debug(" {}".format(input_name))
input_match = daymet_re.match(input_name)
if not input_match:
logging.debug(' Regular expression didn\'t match, skipping')
continue
elif input_match.group('VAR') != input_var:
logging.debug(' Variable didn\'t match, skipping')
continue
year_str = input_match.group('YEAR')
logging.info(" Year: {}".format(year_str))
year_int = int(year_str)
year_days = int(dt.datetime(year_int, 12, 31).strftime('%j'))
if start_dt is not None and year_int < start_dt.year:
logging.debug(' Before start date, skipping')
continue
elif end_dt is not None and year_int > end_dt.year:
logging.debug(' After end date, skipping')
continue
# Build input file path
input_raster = os.path.join(netcdf_ws, input_name)
if not os.path.isfile(input_raster):
logging.debug(
' Input raster doesn\'t exist, skipping {}'.format(
input_raster))
continue
# Build output folder
if daily_flag:
daily_ws = os.path.join(var_ws, 'daily')
if not os.path.isdir(daily_ws):
os.makedirs(daily_ws)
if monthly_flag:
monthly_temp_sum = np.full(
(12, output_shape[0], output_shape[1]), 0, np.float64)
monthly_temp_count = np.full(
(12, output_shape[0], output_shape[1]), 0, np.uint8)
# Read in the DAYMET NetCDF file
input_nc_f = netCDF4.Dataset(input_raster, 'r')
# logging.debug(input_nc_f.variables)
# Check all valid dates in the year
year_dates = _utils.date_range(
dt.datetime(year_int, 1, 1), dt.datetime(year_int + 1, 1, 1))
for date_dt in year_dates:
logging.debug(' {}'.format(date_dt.date()))
# if start_dt is not None and date_dt < start_dt:
# logging.debug(
# ' {} - before start date, skipping'.format(
# date_dt.date()))
# continue
# elif end_dt is not None and date_dt > end_dt:
# logging.debug(' {} - after end date, skipping'.format(
# date_dt.date()))
# continue
# else:
# logging.info(' {}'.format(date_dt.date()))
doy = int(date_dt.strftime('%j'))
doy_i = range(1, year_days + 1).index(doy)
month_i = date_dt.month - 1
# Arrays are being read as masked array with a -9999 fill value
# Convert to basic numpy array arrays with nan values
try:
input_ma = input_nc_f.variables[input_var][
doy_i, yi: yi + output_rows, xi: xi + output_cols]
except IndexError:
logging.info(' date not in netcdf, skipping')
continue
input_nodata = float(input_ma.fill_value)
output_array = input_ma.data.astype(np.float32)
output_array[output_array == input_nodata] = np.nan
output_mask = np.isfinite(output_array)
# Convert Kelvin to Celsius
if input_var in ['tmax', 'tmin']:
output_array -= 273.15
# Save values
if daily_flag:
daily_sum[doy_i, :, :] += output_array
daily_count[doy_i, :, :] += output_mask
if monthly_flag:
monthly_temp_sum[month_i, :, :] += output_array
monthly_temp_count[month_i, :, :] += output_mask
if annual_flag:
annual_sum[:, :] += output_array
annual_count[:, :] += output_mask
# Cleanup
# del input_ds, input_array
del input_ma, output_array, output_mask
# Compute mean monthly for the year
if monthly_flag:
# Sum precipitation
if input_var == 'prcp':
monthly_sum += monthly_temp_sum
else:
monthly_sum += monthly_temp_sum / monthly_temp_count
# Is this the right count?
monthly_count += np.any(monthly_temp_count, axis=0)
del monthly_temp_sum, monthly_temp_count
input_nc_f.close()
del input_nc_f
# Save the projected climatology arrays
if daily_flag:
for doy_i in range(daily_sum.shape[0]):
daily_name = daily_fmt.format(
var=output_var, start=start_year, end=end_year,
doy=doy_i + 1)
daily_path = os.path.join(daily_ws, daily_name)
drigo.array_to_raster(
daily_sum[doy_i, :, :] / daily_count[doy_i, :, :],
daily_path, output_geo=output_geo,
output_proj=daymet_proj, stats_flag=stats_flag)
del daily_sum, daily_count
if monthly_flag:
for month_i in range(monthly_sum.shape[0]):
monthly_name = monthly_fmt.format(
var=output_var, start=start_year, end=end_year,
month=month_i + 1)
monthly_path = os.path.join(var_ws, monthly_name)
drigo.array_to_raster(
monthly_sum[month_i, :, :] / monthly_count[month_i, :, :],
monthly_path, output_geo=output_geo,
output_proj=daymet_proj, stats_flag=stats_flag)
del monthly_sum, monthly_count
if annual_flag:
annual_name = annual_fmt.format(
var=output_var, start=start_year, end=end_year)
annual_path = os.path.join(var_ws, annual_name)
drigo.array_to_raster(
annual_sum / annual_count, annual_path,
output_geo=output_geo, output_proj=daymet_proj,
stats_flag=stats_flag)
del annual_sum, annual_count
logging.debug('\nScript Complete')
def main(ini_path, tile_list=None, overwrite_flag=False, mp_procs=1):
"""Prep Landsat path/row specific data
Parameters
----------
ini_path : str
File path of the input parameters file.
tile_list : list, optional
Landsat path/rows to process (i.e. [p045r043, p045r033]).
This will override the tile list in the INI file.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
mp_procs : int, optional
Number of cores to use (the default is 1).
Returns
-------
None
"""
logging.info('\nPrepare path/row data')
# Open config file
config = python_common.open_ini(ini_path)
# Get input parameters
logging.debug(' Reading Input File')
year = config.getint('INPUTS', 'year')
if tile_list is None:
tile_list = python_common.read_param('tile_list', [], config, 'INPUTS')
project_ws = config.get('INPUTS', 'project_folder')
logging.debug(' Year: {}'.format(year))
logging.debug(' Path/rows: {}'.format(', '.join(tile_list)))
logging.debug(' Project: {}'.format(project_ws))
# study_area_path = config.get('INPUTS', 'study_area_path')
footprint_path = config.get('INPUTS', 'footprint_path')
# For now, assume the UTM zone file is colocated with the footprints shapefile
utm_path = python_common.read_param(
'utm_path',
os.path.join(os.path.dirname(footprint_path),
'wrs2_tile_utm_zones.json'), config, 'INPUTS')
skip_list_path = python_common.read_param('skip_list_path', '', config,
'INPUTS')
landsat_flag = python_common.read_param('landsat_flag', True, config,
'INPUTS')
ledaps_flag = False
dem_flag = python_common.read_param('dem_flag', True, config, 'INPUTS')
nlcd_flag = python_common.read_param('nlcd_flag', True, config, 'INPUTS')
cdl_flag = python_common.read_param('cdl_flag', False, config, 'INPUTS')
landfire_flag = python_common.read_param('landfire_flag', False, config,
'INPUTS')
field_flag = python_common.read_param('field_flag', False, config,
'INPUTS')
tile_gcs_buffer = python_common.read_param('tile_buffer', 0.25, config)
# Input/output folder and file paths
if landsat_flag:
landsat_input_ws = config.get('INPUTS', 'landsat_input_folder')
else:
landsat_input_ws = None
# if ledaps_flag:
# ledaps_input_ws = config.get('INPUTS', 'ledaps_input_folder')
# else:
# ledaps_input_ws = None
if dem_flag:
dem_input_ws = config.get('INPUTS', 'dem_input_folder')
dem_tile_fmt = config.get('INPUTS', 'dem_tile_fmt')
dem_output_ws = config.get('INPUTS', 'dem_output_folder')
dem_output_name = python_common.read_param('dem_output_name',
'dem.img', config)
# dem_output_name = config.get('INPUTS', 'dem_output_name')
else:
dem_input_ws, dem_tile_fmt = None, None
dem_output_ws, dem_output_name = None, None
if nlcd_flag:
nlcd_input_path = config.get('INPUTS', 'nlcd_input_path')
nlcd_output_ws = config.get('INPUTS', 'nlcd_output_folder')
nlcd_output_fmt = python_common.read_param('nlcd_output_fmt',
'nlcd_{:04d}.img', config)
else:
nlcd_input_path, nlcd_output_ws, nlcd_output_fmt = None, None, None
if cdl_flag:
cdl_input_path = config.get('INPUTS', 'cdl_input_path')
cdl_ag_list = config.get('INPUTS', 'cdl_ag_list')
cdl_ag_list = list(python_common.parse_int_set(cdl_ag_list))
# default_cdl_ag_list = range(1,62) + range(66,78) + range(204,255)
# cdl_ag_list = python_common.read_param(
# 'cdl_ag_list', default_cdl_ag_list, config)
# cdl_ag_list = list(map(int, cdl_ag_list))
# cdl_non_ag_list = python_common.read_param(
# 'cdl_non_ag_list', [], config)
cdl_output_ws = config.get('INPUTS', 'cdl_output_folder')
cdl_output_fmt = python_common.read_param('cdl_output_fmt',
'cdl_{:04d}.img', config)
cdl_ag_output_fmt = python_common.read_param('cdl_ag_output_fmt',
'cdl_ag_{:04d}.img',
config)
else:
cdl_input_path, cdl_ag_list = None, None
cdl_output_ws, cdl_output_fmt, cdl_ag_output_fmt = None, None, None
if landfire_flag:
landfire_input_path = config.get('INPUTS', 'landfire_input_path')
landfire_ag_list = config.get('INPUTS', 'landfire_ag_list')
landfire_ag_list = list(python_common.parse_int_set(landfire_ag_list))
# default_landfire_ag_list = range(3960,4000)
# landfire_ag_list = python_common.read_param(
# 'landfire_ag_list', default_landfire_ag_list, config)
# landfire_ag_list = list(map(int, landfire_ag_list))
landfire_output_ws = config.get('INPUTS', 'landfire_output_folder')
landfire_output_fmt = python_common.read_param('landfire_output_fmt',
'landfire_{:04d}.img',
config)
landfire_ag_output_fmt = python_common.read_param(
'landfire_ag_output_fmt', 'landfire_ag_{:04d}.img', config)
else:
landfire_input_path, landfire_ag_list = None, None
landfire_output_ws = None
landfire_output_fmt, landfire_ag_output_fmt = None, None
if field_flag:
field_input_path = config.get('INPUTS', 'field_input_path')
field_output_ws = config.get('INPUTS', 'field_output_folder')
field_output_fmt = python_common.read_param('field_output_fmt',
'fields_{:04d}.img',
config)
else:
field_input_path = None
field_output_ws, field_output_fmt = None, None
# File/folder names
orig_data_folder_name = 'ORIGINAL_DATA'
# Check inputs folders/paths
logging.info('\nChecking input folders/files')
file_check(footprint_path)
file_check(utm_path)
if landsat_flag:
folder_check(landsat_input_ws)
# if ledaps_flag:
# folder_check(ledaps_input_ws)
if dem_flag:
folder_check(dem_input_ws)
if nlcd_flag:
file_check(nlcd_input_path)
if cdl_flag:
file_check(cdl_input_path)
if landfire_flag:
# Landfire will likely be an ESRI grid (set as a folder)
if not (os.path.isdir(landfire_input_path)
or os.path.isfile(landfire_input_path)):
logging.error('\n {} does not exist'.format(landfire_input_path))
if field_flag:
file_check(field_input_path)
if skip_list_path:
file_check(skip_list_path)
# Build output folders
if not os.path.isdir(project_ws):
os.makedirs(project_ws)
if dem_flag and not os.path.isdir(dem_output_ws):
os.makedirs(dem_output_ws)
if nlcd_flag and not os.path.isdir(nlcd_output_ws):
os.makedirs(nlcd_output_ws)
if cdl_flag and not os.path.isdir(cdl_output_ws):
os.makedirs(cdl_output_ws)
if landfire_flag and not os.path.isdir(landfire_output_ws):
os.makedirs(landfire_output_ws)
if field_flag and not os.path.isdir(field_output_ws):
os.makedirs(field_output_ws)
# For now assume path/row are two digit numbers
tile_fmt = 'p{:03d}r{:03d}'
tile_re = re.compile('p(\d{3})r(\d{3})')
image_re = re.compile(
'^(LT04|LT05|LE07|LC08)_(\d{3})(\d{3})_(\d{4})(\d{2})(\d{2})')
snap_cs = 30
snap_xmin, snap_ymin = (15, 15)
# Set snap environment parameters
env = drigo.env
env.cellsize = snap_cs
env.snap_xmin, env.snap_ymin = snap_xmin, snap_ymin
# Use WGSS84 (EPSG 4326) for GCS spatial reference
# Could also use NAD83 (EPSG 4269)
# gcs_epsg = 4326
# gcs_osr = epsg_osr(4326)
# gcs_proj = osr_proj(gcs_osr)
# Landsat Footprints (WRS2 Descending Polygons)
logging.debug('\nFootprint (WRS2 descending should be GCS84):')
tile_gcs_osr = drigo.feature_path_osr(footprint_path)
logging.debug(' OSR: {}'.format(tile_gcs_osr))
# Doublecheck that WRS2 descending shapefile is GCS84
# if tile_gcs_osr != epsg_osr(4326):
# logging.error(' WRS2 is not GCS84')
# sys.exit()
# Get geometry for each path/row
tile_gcs_wkt_dict = path_row_wkt_func(footprint_path,
path_field='PATH',
row_field='ROW')
# Get UTM zone for each path/row
# DEADBEEF - Using "eval" is considered unsafe and should be changed
tile_utm_zone_dict = eval(open(utm_path, 'r').read())
# Project study area geometry to GCS coordinates
# logging.debug('\nStudy area')
# study_area_geom = feature_path_geom_union(study_area_path)
# study_area_gcs_geom = study_area_geom.Clone()
# study_area_gcs_geom.TransformTo(tile_gcs_osr)
# Get list of all intersecting Landsat path/rows
# logging.info('\nLandsat path/rows')
# tile_list = []
# for tile_name, tile_gcs_wkt in tile_gcs_wkt_dict.items():
# tile_gcs_geom = ogr.CreateGeometryFromWkt(tile_gcs_wkt)
# if tile_gcs_geom.Intersects(study_area_gcs_geom):
# tile_list.append(tile_name)
# for tile_name in sorted(tile_list):
# logging.debug(' {}'.format(tile_name))
# Check that each path/row extent and UTM zone exist
logging.info('\nChecking path/row list against footprint shapefile')
for tile_name in sorted(tile_list):
if tile_name not in tile_gcs_wkt_dict.keys():
logging.error(
' {} feature not in footprint shapefile'.format(tile_name))
continue
elif tile_name not in tile_utm_zone_dict.keys():
logging.error(
' {} UTM zone not in footprint shapefile'.format(tile_name))
continue
elif tile_utm_zone_dict[tile_name] == 0:
logging.error((' UTM zone is not set for {} in ' +
'footprint shapefile').format(tile_name))
continue
# Build output folders for each path/row
logging.info('\nBuilding path/row folders')
for tile_name in tile_list:
logging.debug(' {} {}'.format(year, tile_name))
tile_output_ws = os.path.join(project_ws, str(year), tile_name)
if ((landsat_flag or ledaps_flag)
and not os.path.isdir(tile_output_ws)):
os.makedirs(tile_output_ws)
if (dem_flag
and not os.path.isdir(os.path.join(dem_output_ws, tile_name))):
os.makedirs(os.path.join(dem_output_ws, tile_name))
if (nlcd_flag and
not os.path.isdir(os.path.join(nlcd_output_ws, tile_name))):
os.makedirs(os.path.join(nlcd_output_ws, tile_name))
if (cdl_flag
and not os.path.isdir(os.path.join(cdl_output_ws, tile_name))):
os.makedirs(os.path.join(cdl_output_ws, tile_name))
if (landfire_flag and not os.path.isdir(
os.path.join(landfire_output_ws, tile_name))):
os.makedirs(os.path.join(landfire_output_ws, tile_name))
if (field_flag and
not os.path.isdir(os.path.join(field_output_ws, tile_name))):
os.makedirs(os.path.join(field_output_ws, tile_name))
# Read skip list
if (landsat_flag or ledaps_flag) and skip_list_path:
logging.debug('\nReading scene skiplist')
with open(skip_list_path) as skip_list_f:
skip_list = skip_list_f.readlines()
skip_list = [
scene.strip() for scene in skip_list
if image_re.match(scene.strip())
]
else:
logging.debug('\nSkip list not set in INI')
skip_list = []
# Copy and unzip raw Landsat scenes
# Use these for thermal band, MTL file (scene time), and to run FMask
if landsat_flag:
logging.info('\nExtract raw Landsat scenes')
# Process each path/row
extract_targz_list = []
for tile_name in tile_list:
tile_output_ws = os.path.join(project_ws, str(year), tile_name)
# path/row as strings with leading zeros
path, row = map(str, tile_re.match(tile_name).groups())
tile_input_ws = os.path.join(landsat_input_ws, path, row,
str(year))
if not os.path.isdir(tile_input_ws):
continue
logging.info(' {} {}'.format(year, tile_name))
# Process each tar.gz file
for input_name in sorted(os.listdir(tile_input_ws)):
if (not image_re.match(input_name)
and not input_name.endswith('.tar.gz')):
continue
# Get Landsat scene ID from tar.gz file name
# DEADBEEF - For now this is the EE scene ID, but it could be
# changed to the full collection 1 ID
scene_id = input_name.split('.')[0]
# Output workspace
image_output_ws = os.path.join(tile_output_ws, scene_id)
orig_data_ws = os.path.join(image_output_ws,
orig_data_folder_name)
if skip_list and scene_id in skip_list:
logging.debug(' {} - Skipping scene'.format(scene_id))
# DEADBEEF - Should the script always remove the scene
# if it is in the skip list?
# Maybe only if overwrite is set?
if os.path.isdir(image_output_ws):
# input('Press ENTER to delete {}'.format(scene_id))
shutil.rmtree(image_output_ws)
continue
# If orig_data_ws doesn't exist, don't check images
if not os.path.isdir(orig_data_ws):
os.makedirs(orig_data_ws)
elif (not overwrite_flag
and landsat_files_check(image_output_ws)):
continue
# Extract Landsat tar.gz file
input_path = os.path.join(tile_input_ws, input_name)
print(orig_data_ws)
# sys.exit()
if mp_procs > 1:
extract_targz_list.append([input_path, orig_data_ws])
else:
python_common.extract_targz_func(input_path, orig_data_ws)
# # Use a command line call
# input_path = os.path.join(tile_input_ws, input_name)
# if job_i % pbs_jobs != 0:
# job_list.append('tar -zxvf {} -C {} &\n'.format(
# input_path, orig_data_ws))
# else:
# job_list.append('tar -zxvf {} -C {}\n'.format(
# input_path, orig_data_ws))
# # job_list.append('tar -zxvf {} -C {} &\n'.format(
# # input_path, orig_data_ws))
# # job_list.append('wait\n')
# job_i += 1
# Extract Landsat tar.gz files using multiprocessing
if extract_targz_list:
pool = mp.Pool(mp_procs)
results = pool.map(python_common.extract_targz_mp,
extract_targz_list,
chunksize=1)
pool.close()
pool.join()
del results, pool
# Get projected extent for each path/row
# This should probably be in a function
if (dem_flag or nlcd_flag or cdl_flag or landfire_flag or field_flag):
tile_utm_extent_dict = gcs_to_utm_dict(tile_list, tile_utm_zone_dict,
tile_gcs_osr, tile_gcs_wkt_dict,
tile_gcs_buffer, snap_xmin,
snap_ymin, snap_cs)
# Mosaic DEM tiles for each path/row
if dem_flag:
logging.info('\nBuild DEM for each path/row')
mosaic_mp_list = []
for tile_name in tile_list:
# Output folder and path
tile_output_path = os.path.join(dem_output_ws, tile_name,
dem_output_name)
if not overwrite_flag and os.path.isfile(tile_output_path):
logging.debug(' {} already exists, skipping'.format(
os.path.basename(tile_output_path)))
continue
logging.info(' {}'.format(tile_name))
# Get the path/row geometry in GCS for selecting intersecting tiles
tile_gcs_geom = ogr.CreateGeometryFromWkt(
tile_gcs_wkt_dict[tile_name])
# Apply a small buffer (in degrees) to the extent
# DEADBEEF - Buffer fails if GDAL is not built with GEOS support
# tile_gcs_geom = tile_gcs_geom.Buffer(tile_gcs_buffer)
tile_gcs_extent = drigo.Extent(tile_gcs_geom.GetEnvelope())
tile_gcs_extent = tile_gcs_extent.ogrenv_swap()
tile_gcs_extent.buffer_extent(tile_gcs_buffer)
# tile_gcs_extent.ymin, tile_gcs_extent.xmax = tile_gcs_extent.xmax, tile_gcs_extent.ymin
# Offsets are needed since tile name is upper left corner of tile
# Tile n36w120 spans -120 <-> -119 and 35 <-> 36
lon_list = range(
int(tile_gcs_extent.xmin) - 1, int(tile_gcs_extent.xmax))
lat_list = range(
int(tile_gcs_extent.ymin) + 1,
int(tile_gcs_extent.ymax) + 2)
# Get list of DEM tile rasters
dem_tile_list = []
for lat, lon in itertools.product(lat_list, lon_list):
# Convert sign of lat/lon to letter
lat = ('n' + '{:02d}'.format(abs(lat)) if lat >= 0 else 's' +
'{:02d}'.format(abs(lat)))
lon = ('w' + '{:03d}'.format(abs(lon)) if lon < 0 else 'e' +
'{:03d}'.format(abs(lon)))
dem_tile_path = os.path.join(dem_input_ws,
dem_tile_fmt.format(lat, lon))
if os.path.isfile(dem_tile_path):
dem_tile_list.append(dem_tile_path)
if not dem_tile_list:
logging.warning(' WARNING: No DEM tiles were selected')
continue
# Mosaic tiles using mosaic function
tile_utm_osr = drigo.epsg_osr(32600 +
int(tile_utm_zone_dict[tile_name]))
tile_utm_proj = drigo.epsg_proj(32600 +
int(tile_utm_zone_dict[tile_name]))
tile_utm_extent = tile_utm_extent_dict[tile_name]
tile_utm_ullr = tile_utm_extent.ul_lr_swap()
# Mosaic, clip, project using custom function
if mp_procs > 1:
mosaic_mp_list.append([
dem_tile_list, tile_output_path, tile_utm_proj, snap_cs,
tile_utm_extent
])
else:
drigo.mosaic_tiles(dem_tile_list, tile_output_path,
tile_utm_osr, snap_cs, tile_utm_extent)
# Cleanup
del tile_output_path
del tile_gcs_geom, tile_gcs_extent, tile_utm_extent
del tile_utm_osr, tile_utm_proj
del lon_list, lat_list, dem_tile_list
# Mosaic DEM rasters using multiprocessing
if mosaic_mp_list:
pool = mp.Pool(mp_procs)
results = pool.map(mosaic_tiles_mp, mosaic_mp_list, chunksize=1)
pool.close()
pool.join()
del results, pool
# Project/clip NLCD for each path/row
if nlcd_flag:
logging.info('\nBuild NLCD for each path/row')
project_mp_list = []
for tile_name in tile_list:
nlcd_output_path = os.path.join(nlcd_output_ws, tile_name,
nlcd_output_fmt.format(year))
if not overwrite_flag and os.path.isfile(nlcd_output_path):
logging.debug(' {} already exists, skipping'.format(
os.path.basename(nlcd_output_path)))
continue
logging.info(' {}'.format(tile_name))
# Set the nodata value on the NLCD raster if it is not set
nlcd_ds = gdal.Open(nlcd_input_path, 0)
nlcd_band = nlcd_ds.GetRasterBand(1)
nlcd_nodata = nlcd_band.GetNoDataValue()
nlcd_ds = None
if nlcd_nodata is None:
nlcd_nodata = 255
# Clip and project
tile_utm_osr = drigo.epsg_osr(32600 +
int(tile_utm_zone_dict[tile_name]))
tile_utm_proj = drigo.epsg_proj(32600 +
int(tile_utm_zone_dict[tile_name]))
tile_utm_extent = tile_utm_extent_dict[tile_name]
tile_utm_ullr = tile_utm_extent.ul_lr_swap()
if mp_procs > 1:
project_mp_list.append([
nlcd_input_path, nlcd_output_path,
gdal.GRA_NearestNeighbour, tile_utm_proj, snap_cs,
tile_utm_extent, nlcd_nodata
])
else:
drigo.project_raster(nlcd_input_path, nlcd_output_path,
gdal.GRA_NearestNeighbour, tile_utm_osr,
snap_cs, tile_utm_extent, nlcd_nodata)
# Cleanup
del nlcd_output_path
del nlcd_ds, nlcd_band, nlcd_nodata
del tile_utm_osr, tile_utm_proj, tile_utm_extent
# Project NLCD rasters using multiprocessing
if project_mp_list:
pool = mp.Pool(mp_procs)
results = pool.map(drigo.project_raster_mp,
project_mp_list,
chunksize=1)
pool.close()
pool.join()
del results, pool
# Project/clip CDL for each path/row
if cdl_flag:
logging.info('\nBuild CDL for each path/row')
project_mp_list, remap_mp_list = [], []
for tile_name in tile_list:
cdl_output_path = os.path.join(cdl_output_ws, tile_name,
cdl_output_fmt.format(year))
cdl_ag_output_path = os.path.join(cdl_output_ws, tile_name,
cdl_ag_output_fmt.format(year))
if not os.path.isfile(cdl_input_path):
logging.error('\n\n {} does not exist'.format(cdl_input_path))
sys.exit()
if not overwrite_flag and os.path.isfile(cdl_output_path):
logging.debug(' {} already exists, skipping'.format(
os.path.basename(cdl_output_path)))
continue
logging.info(' {}'.format(tile_name))
# Set the nodata value on the CDL raster if it is not set
cdl_ds = gdal.Open(cdl_input_path, 0)
cdl_band = cdl_ds.GetRasterBand(1)
cdl_nodata = cdl_band.GetNoDataValue()
cdl_ds = None
if cdl_nodata is None:
cdl_nodata = 255
# Clip and project
tile_utm_osr = drigo.epsg_osr(32600 +
int(tile_utm_zone_dict[tile_name]))
tile_utm_proj = drigo.epsg_proj(32600 +
int(tile_utm_zone_dict[tile_name]))
tile_utm_extent = tile_utm_extent_dict[tile_name]
if mp_procs > 1:
project_mp_list.append([
cdl_input_path, cdl_output_path, gdal.GRA_NearestNeighbour,
tile_utm_proj, snap_cs, tile_utm_extent, cdl_nodata
])
remap_mp_list.append(
[cdl_output_path, cdl_ag_output_path, cdl_ag_list])
else:
drigo.project_raster(cdl_input_path, cdl_output_path,
gdal.GRA_NearestNeighbour, tile_utm_osr,
snap_cs, tile_utm_extent, cdl_nodata)
# Build a mask of CDL ag lands
remap_mask_func(cdl_output_path, cdl_ag_output_path,
cdl_ag_list)
# Cleanup
del cdl_output_path
del cdl_ds, cdl_band, cdl_nodata
del tile_utm_osr, tile_utm_proj, tile_utm_extent
# Project CDL rasters using multiprocessing
if project_mp_list:
pool = mp.Pool(mp_procs)
results = pool.map(drigo.project_raster_mp,
project_mp_list,
chunksize=1)
pool.close()
pool.join()
del results, pool
if remap_mp_list:
pool = mp.Pool(mp_procs)
results = pool.map(remap_mask_mp, remap_mp_list, chunksize=1)
pool.close()
pool.join()
del results, pool
# Project/clip LANDFIRE for each path/row
if landfire_flag:
logging.info('\nBuild LANDFIRE for each path/row')
project_mp_list, remap_mp_list = [], []
for tile_name in tile_list:
landfire_output_path = os.path.join(
landfire_output_ws, tile_name,
landfire_output_fmt.format(year))
landfire_ag_output_path = os.path.join(
landfire_output_ws, tile_name,
landfire_ag_output_fmt.format(year))
if not overwrite_flag and os.path.isfile(landfire_output_path):
logging.debug(' {} already exists, skipping'.format(
os.path.basename(landfire_output_path)))
continue
logging.info(' {}'.format(tile_name))
# Set the nodata value on the LANDFIRE raster if it is not set
# landfire_ds = gdal.Open(landfire_input_path, 0)
# landfire_band = landfire_ds.GetRasterBand(1)
# landfire_nodata = landfire_band.GetNoDataValue()
# landfire_ds = None
# if landfire_nodata is None:
# landfire_nodata = 32767
# del landfire_ds, landfire_band
landfire_nodata = 32767
# Clip and project
tile_utm_osr = drigo.epsg_osr(32600 +
int(tile_utm_zone_dict[tile_name]))
tile_utm_proj = drigo.epsg_proj(32600 +
int(tile_utm_zone_dict[tile_name]))
tile_utm_extent = tile_utm_extent_dict[tile_name]
if mp_procs > 1:
project_mp_list.append([
landfire_input_path, landfire_output_path,
gdal.GRA_NearestNeighbour, tile_utm_proj, snap_cs,
tile_utm_extent, landfire_nodata
])
remap_mp_list.append([
landfire_output_path, landfire_ag_output_path,
landfire_ag_list
])
else:
drigo.project_raster(landfire_input_path, landfire_output_path,
gdal.GRA_NearestNeighbour, tile_utm_osr,
snap_cs, tile_utm_extent, landfire_nodata)
# Build a mask of LANDFIRE ag lands
remap_mask_func(landfire_output_path, landfire_ag_output_path,
landfire_ag_list)
# Cleanup
del landfire_output_path
del tile_utm_osr, tile_utm_proj, tile_utm_extent
# Project LANDFIRE rasters using multiprocessing
if project_mp_list:
pool = mp.Pool(mp_procs)
results = pool.map(drigo.project_raster_mp,
project_mp_list,
chunksize=1)
pool.close()
pool.join()
del results, pool
if remap_mp_list:
pool = mp.Pool(mp_procs)
results = pool.map(remap_mask_mp, remap_mp_list, chunksize=1)
pool.close()
pool.join()
del results, pool
# Convert field shapefiles to raster
if field_flag:
logging.info('\nBuild field rasters for each path/row')
for tile_name in tile_list:
logging.info(' {}'.format(tile_name))
tile_output_ws = os.path.join(field_output_ws, tile_name)
# Shapefile paths
field_proj_name = (
os.path.splitext(field_output_fmt.format(year))[0] +
"_wgs84z{}.shp".format(tile_utm_zone_dict[tile_name]))
field_proj_path = os.path.join(tile_output_ws, field_proj_name)
field_output_path = os.path.join(tile_output_ws,
field_output_fmt.format(year))
if not overwrite_flag and os.path.isfile(field_output_path):
logging.debug(' {} already exists, skipping'.format(
os.path.basename(field_output_path)))
continue
# The ogr2ogr spatial query is in the input spatial reference
# Project the path/row extent to the field osr/proj
field_input_osr = drigo.feature_path_osr(field_input_path)
tile_utm_osr = drigo.epsg_osr(32600 +
int(tile_utm_zone_dict[tile_name]))
# field_input_proj = drigo.osr_proj(field_input_osr)
# tile_utm_proj = drigo.osr_proj(tile_utm_osr)
field_tile_extent = drigo.project_extent(
tile_utm_extent_dict[tile_name], tile_utm_osr, field_input_osr,
30)
# Project shapefile to the path/row zone
# Clipping requires GDAL to be built with GEOS support
subprocess.call(
[
'ogr2ogr', '-t_srs', 'EPSG:326{}'.format(
tile_utm_zone_dict[tile_name]), '-f', 'ESRI Shapefile',
'-overwrite'
] + ['-spat'] + list(map(str, field_tile_extent)) +
['-clipdst'] +
list(map(str, tile_utm_extent_dict[tile_name])) +
# ['-clipdst'] + list(map(str, tile_utm_extent_dict[tile_name])) +
# ['-clipsrc'] + list(map(str, field_tile_extent)) +
# ['-clipsrc'] + list(map(str, field_tile_extent)) +
[field_proj_path, field_input_path])
# Convert shapefile to raster
field_mem_ds = drigo.polygon_to_raster_ds(
field_proj_path,
nodata_value=0,
burn_value=1,
output_osr=tile_utm_osr,
output_extent=tile_utm_extent_dict[tile_name])
field_output_driver = drigo.raster_driver(field_output_path)
if field_output_path.lower().endswith('.img'):
field_output_ds = field_output_driver.CreateCopy(
field_output_path, field_mem_ds, 0, ['COMPRESS=YES'])
else:
field_output_ds = field_output_driver.CreateCopy(
field_output_path, field_mem_ds, 0)
field_output_ds, field_mem_ds = None, None
# Remove field shapefile
# try:
# remove_file(field_proj_path)
# except:
# pass
# Cleanup
del tile_utm_osr, field_tile_extent, field_input_osr
# del tile_utm_proj, field_input_proj
del field_proj_name, field_proj_path, field_output_path
logging.debug('\nScript complete')
def main(netcdf_ws=os.getcwd(),
ancillary_ws=os.getcwd(),
output_ws=os.getcwd(),
variables=['prcp'],
start_date=None,
end_date=None,
extent_path=None,
output_extent=None,
stats_flag=True,
overwrite_flag=False):
"""Extract DAYMET temperature
Parameters
----------
netcdf_ws : str
Folder of DAYMET netcdf files.
ancillary_ws : str
Folder of ancillary rasters.
output_ws : str
Folder of output rasters.
variables : list, optional
DAYMET variables to download ('prcp', 'srad', 'vp', 'tmmn', 'tmmx').
Set as ['all'] to process all variables.
start_date : str, optional
ISO format date (YYYY-MM-DD).
end_date : str, optional
ISO format date (YYYY-MM-DD).
extent_path : str, optional
File path defining the output extent.
output_extent : list, optional
Decimal degrees values defining output extent.
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nExtracting DAYMET variables')
# If a date is not set, process 2015
try:
start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d')
logging.debug(' Start date: {}'.format(start_dt))
except:
start_dt = dt.datetime(2015, 1, 1)
logging.info(' Start date: {}'.format(start_dt))
try:
end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d')
logging.debug(' End date: {}'.format(end_dt))
except:
end_dt = dt.datetime(2015, 12, 31)
logging.info(' End date: {}'.format(end_dt))
# Get DAYMET spatial reference from an ancillary raster
mask_raster = os.path.join(ancillary_ws, 'daymet_mask.img')
daymet_re = re.compile('daymet_v3_(?P<VAR>\w+)_(?P<YEAR>\d{4})_na.nc4$')
# DAYMET rasters to extract
var_full_list = ['prcp', 'srad', 'vp', 'tmmn', 'tmmx']
if not variables:
logging.error('\nERROR: variables parameter is empty\n')
sys.exit()
elif type(variables) is not list:
# DEADBEEF - I could try converting comma separated strings to lists?
logging.warning('\nERROR: variables parameter must be a list\n')
sys.exit()
elif 'all' in variables:
logging.error('\nDownloading all variables\n {}'.format(
','.join(var_full_list)))
var_list = var_full_list[:]
elif not set(variables).issubset(set(var_full_list)):
logging.error(
'\nERROR: variables parameter is invalid\n {}'.format(variables))
sys.exit()
else:
var_list = variables[:]
# DAYMET band name dictionary
# daymet_band_dict = dict()
# daymet_band_dict['prcp'] = 'precipitation_amount'
# daymet_band_dict['srad'] = 'surface_downwelling_shortwave_flux_in_air'
# daymet_band_dict['sph'] = 'specific_humidity'
# daymet_band_dict['tmin'] = 'air_temperature'
# daymet_band_dict['tmax'] = 'air_temperature'
# Get extent/geo from mask raster
daymet_ds = gdal.Open(mask_raster)
daymet_osr = drigo.raster_ds_osr(daymet_ds)
daymet_proj = drigo.osr_proj(daymet_osr)
daymet_cs = drigo.raster_ds_cellsize(daymet_ds, x_only=True)
daymet_extent = drigo.raster_ds_extent(daymet_ds)
daymet_geo = daymet_extent.geo(daymet_cs)
daymet_x, daymet_y = daymet_extent.origin()
daymet_ds = None
logging.debug(' Projection: {}'.format(daymet_proj))
logging.debug(' Cellsize: {}'.format(daymet_cs))
logging.debug(' Geo: {}'.format(daymet_geo))
logging.debug(' Extent: {}'.format(daymet_extent))
logging.debug(' Origin: {} {}'.format(daymet_x, daymet_y))
# Subset data to a smaller extent
if output_extent is not None:
logging.info('\nComputing subset extent & geo')
logging.debug(' Extent: {}'.format(output_extent))
# Assume input extent is in decimal degrees
output_extent = drigo.project_extent(drigo.Extent(output_extent),
drigo.epsg_osr(4326), daymet_osr,
0.001)
output_extent = drigo.intersect_extents([daymet_extent, output_extent])
output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs)
output_geo = output_extent.geo(daymet_cs)
logging.debug(' Geo: {}'.format(output_geo))
logging.debug(' Extent: {}'.format(output_extent))
elif extent_path is not None:
logging.info('\nComputing subset extent & geo')
if extent_path.lower().endswith('.shp'):
output_extent = drigo.feature_path_extent(extent_path)
extent_osr = drigo.feature_path_osr(extent_path)
extent_cs = None
else:
output_extent = drigo.raster_path_extent(extent_path)
extent_osr = drigo.raster_path_osr(extent_path)
extent_cs = drigo.raster_path_cellsize(extent_path, x_only=True)
output_extent = drigo.project_extent(output_extent, extent_osr,
daymet_osr, extent_cs)
output_extent = drigo.intersect_extents([daymet_extent, output_extent])
output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs)
output_geo = output_extent.geo(daymet_cs)
logging.debug(' Geo: {}'.format(output_geo))
logging.debug(' Extent: {}'.format(output_extent))
else:
output_extent = daymet_extent.copy()
output_geo = daymet_geo[:]
# output_shape = output_extent.shape(cs=daymet_cs)
xi, yi = drigo.array_geo_offsets(daymet_geo, output_geo, daymet_cs)
output_rows, output_cols = output_extent.shape(daymet_cs)
logging.debug(' Shape: {} {}'.format(output_rows, output_cols))
logging.debug(' Offsets: {} {} (x y)'.format(xi, yi))
# Process each variable
for input_var in var_list:
logging.info("\nVariable: {}".format(input_var))
# Rename variables to match cimis
if input_var == 'prcp':
output_var = 'ppt'
else:
output_var = input_var
# Build output folder
var_ws = os.path.join(output_ws, output_var)
if not os.path.isdir(var_ws):
os.makedirs(var_ws)
# Process each file in the input workspace
for input_name in sorted(os.listdir(netcdf_ws)):
logging.debug("{}".format(input_name))
input_match = daymet_re.match(input_name)
if not input_match:
logging.debug(' Regular expression didn\'t match, skipping')
continue
elif input_match.group('VAR') != input_var:
logging.debug(' Variable didn\'t match, skipping')
continue
year_str = input_match.group('YEAR')
logging.info(" Year: {}".format(year_str))
year_int = int(year_str)
year_days = int(dt.datetime(year_int, 12, 31).strftime('%j'))
if start_dt is not None and year_int < start_dt.year:
logging.debug(' Before start date, skipping')
continue
elif end_dt is not None and year_int > end_dt.year:
logging.debug(' After end date, skipping')
continue
# Build input file path
input_raster = os.path.join(netcdf_ws, input_name)
# if not os.path.isfile(input_raster):
# logging.debug(
# ' Input raster doesn\'t exist, skipping {}'.format(
# input_raster))
# continue
# Build output folder
output_year_ws = os.path.join(var_ws, year_str)
if not os.path.isdir(output_year_ws):
os.makedirs(output_year_ws)
# Read in the DAYMET NetCDF file
input_nc_f = netCDF4.Dataset(input_raster, 'r')
# logging.debug(input_nc_f.variables)
# Check all valid dates in the year
year_dates = _utils.date_range(dt.datetime(year_int, 1, 1),
dt.datetime(year_int + 1, 1, 1))
for date_dt in year_dates:
if start_dt is not None and date_dt < start_dt:
logging.debug(' {} - before start date, skipping'.format(
date_dt.date()))
continue
elif end_dt is not None and date_dt > end_dt:
logging.debug(' {} - after end date, skipping'.format(
date_dt.date()))
continue
else:
logging.info(' {}'.format(date_dt.date()))
output_path = os.path.join(
output_year_ws,
'{}_{}_daymet.img'.format(output_var,
date_dt.strftime('%Y%m%d')))
if os.path.isfile(output_path):
logging.debug(' {}'.format(output_path))
if not overwrite_flag:
logging.debug(' File already exists, skipping')
continue
else:
logging.debug(
' File already exists, removing existing')
os.remove(output_path)
doy = int(date_dt.strftime('%j'))
doy_i = range(1, year_days + 1).index(doy)
# Arrays are being read as masked array with a fill value of -9999
# Convert to basic numpy array arrays with nan values
try:
input_ma = input_nc_f.variables[input_var][doy_i, yi:yi +
output_rows,
xi:xi +
output_cols]
except IndexError:
logging.info(' date not in netcdf, skipping')
continue
input_nodata = float(input_ma.fill_value)
output_array = input_ma.data.astype(np.float32)
output_array[output_array == input_nodata] = np.nan
# Convert Kelvin to Celsius
if input_var in ['tmax', 'tmin']:
output_array -= 273.15
# Save the array as 32-bit floats
drigo.array_to_raster(output_array.astype(np.float32),
output_path,
output_geo=output_geo,
output_proj=daymet_proj,
stats_flag=stats_flag)
del input_ma, output_array
input_nc_f.close()
del input_nc_f
logging.debug('\nScript Complete')
def main(extent_path, output_folder, overwrite_flag=False):
"""Download NED tiles that intersect the study_area
Parameters
----------
extent_path : str
File path to study area shapefile.
output_folder : str
Folder path where files will be saved.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
Notes
-----
Script assumes DEM data is in 1x1 WGS84 degree tiles.
Download 10m (1/3 arc-second) or 30m (1 arc-second) versions from:
10m: rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/13/IMG
30m: rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/1/IMG
For this example, only download 30m DEM.
"""
logging.info('\nDownload NED tiles')
site_url = 'rockyftp.cr.usgs.gov'
site_folder = 'vdelivery/Datasets/Staged/Elevation/1/IMG'
# site_url = 'ftp://rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/1/IMG'
zip_fmt = 'n{:02d}w{:03d}.zip'
tile_fmt = 'imgn{:02d}w{:03d}_1.img'
# tile_fmt = 'imgn{:02d}w{:03d}_13.img'
# Use 1 degree snap point and "cellsize" to get 1x1 degree tiles
tile_osr = drigo.epsg_osr(4326)
tile_x, tile_y, tile_cs = 0, 0, 1
buffer_cells = 0
# Error checking
if not os.path.isfile(extent_path):
logging.error('\nERROR: The input_path does not exist\n')
return False
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
# Check that input is a shapefile
# Get the extent of each feature
lat_lon_list = []
shp_driver = ogr.GetDriverByName('ESRI Shapefile')
input_ds = shp_driver.Open(extent_path, 1)
input_osr = drigo.feature_ds_osr(input_ds)
input_layer = input_ds.GetLayer()
input_ftr = input_layer.GetNextFeature()
while input_ftr:
input_geom = input_ftr.GetGeometryRef()
input_extent = drigo.Extent(input_geom.GetEnvelope())
input_extent = input_extent.ogrenv_swap()
input_ftr = input_layer.GetNextFeature()
logging.debug('Input Extent: {}'.format(input_extent))
# Project study area extent to input raster coordinate system
output_extent = drigo.project_extent(input_extent, input_osr, tile_osr)
logging.debug('Output Extent: {}'.format(output_extent))
# Extent needed to select 1x1 degree tiles
tile_extent = output_extent.copy()
tile_extent.adjust_to_snap('EXPAND', tile_x, tile_y, tile_cs)
logging.debug('Tile Extent: {}'.format(tile_extent))
# Get list of avaiable tiles that intersect the extent
lat_lon_list.extend([
(lat, -lon)
for lon in range(int(tile_extent.xmin), int(tile_extent.xmax))
for lat in range(int(tile_extent.ymax), int(tile_extent.ymin), -1)
])
lat_lon_list = sorted(list(set(lat_lon_list)))
# Attempt to download the tiles
logging.info('')
for lat_lon in lat_lon_list:
logging.info('Tile: {}'.format(lat_lon))
zip_name = zip_fmt.format(*lat_lon)
zip_url = '/'.join([site_url, site_folder, zip_name])
zip_path = os.path.join(output_folder, zip_name)
tile_name = tile_fmt.format(*lat_lon)
tile_path = os.path.join(output_folder, tile_name)
logging.debug(' {}'.format(zip_url))
logging.debug(' {}'.format(zip_path))
if os.path.isfile(tile_path) and not overwrite_flag:
logging.debug(' skipping')
continue
_utils.ftp_download(site_url, site_folder, zip_name, zip_path)
logging.debug(' extracting')
try:
zip_f = zipfile.ZipFile(zip_path)
zip_f.extract(tile_name, output_folder)
zip_f.close()
except Exception as e:
logging.info(' Unhandled exception: {}'.format(e))
try:
os.remove(zip_path)
except Exception as e:
logging.info(' Unhandled exception: {}'.format(e))
def main(start_dt,
end_dt,
netcdf_ws,
ancillary_ws,
output_ws,
extent_path=None,
output_extent=None,
stats_flag=True,
overwrite_flag=False):
"""Extract DAYMET temperature
Parameters
----------
start_dt : datetime
Start date.
end_dt : datetime
End date.
netcdf_ws : str
Folder of DAYMET netcdf files.
ancillary_ws : str
Folder of ancillary rasters.
output_ws : str
Folder of output rasters.
extent_path : str, optional
File path defining the output extent.
output_extent : list, optional
Decimal degrees values defining output extent.
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nExtracting DAYMET vapor pressure')
logging.debug(' Start date: {}'.format(start_dt))
logging.debug(' End date: {}'.format(end_dt))
# Get DAYMET spatial reference from an ancillary raster
mask_raster = os.path.join(ancillary_ws, 'daymet_mask.img')
elev_raster = os.path.join(ancillary_ws, 'daymet_elev.img')
daymet_re = re.compile('daymet_v3_(?P<VAR>\w+)_(?P<YEAR>\d{4})_na.nc4$')
# DAYMET band name dictionary
# daymet_band_dict = dict()
# daymet_band_dict['prcp'] = 'precipitation_amount'
# daymet_band_dict['srad'] = 'surface_downwelling_shortwave_flux_in_air'
# daymet_band_dict['sph'] = 'specific_humidity'
# daymet_band_dict['tmin'] = 'air_temperature'
# daymet_band_dict['tmax'] = 'air_temperature'
# Get extent/geo from mask raster
daymet_ds = gdal.Open(mask_raster)
daymet_osr = drigo.raster_ds_osr(daymet_ds)
daymet_proj = drigo.osr_proj(daymet_osr)
daymet_cs = drigo.raster_ds_cellsize(daymet_ds, x_only=True)
daymet_extent = drigo.raster_ds_extent(daymet_ds)
daymet_geo = daymet_extent.geo(daymet_cs)
daymet_x, daymet_y = daymet_extent.origin()
daymet_ds = None
logging.debug(' Projection: {}'.format(daymet_proj))
logging.debug(' Cellsize: {}'.format(daymet_cs))
logging.debug(' Geo: {}'.format(daymet_geo))
logging.debug(' Extent: {}'.format(daymet_extent))
logging.debug(' Origin: {} {}'.format(daymet_x, daymet_y))
# Subset data to a smaller extent
if output_extent is not None:
logging.info('\nComputing subset extent & geo')
logging.debug(' Extent: {}'.format(output_extent))
# Assume input extent is in decimal degrees
output_extent = drigo.project_extent(drigo.Extent(output_extent),
drigo.epsg_osr(4326), daymet_osr,
0.001)
output_extent = drigo.intersect_extents([daymet_extent, output_extent])
output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs)
output_geo = output_extent.geo(daymet_cs)
logging.debug(' Geo: {}'.format(output_geo))
logging.debug(' Extent: {}'.format(output_extent))
elif extent_path is not None:
logging.info('\nComputing subset extent & geo')
if extent_path.lower().endswith('.shp'):
output_extent = drigo.feature_path_extent(extent_path)
extent_osr = drigo.feature_path_osr(extent_path)
extent_cs = None
else:
output_extent = drigo.raster_path_extent(extent_path)
extent_osr = drigo.raster_path_osr(extent_path)
extent_cs = drigo.raster_path_cellsize(extent_path, x_only=True)
output_extent = drigo.project_extent(output_extent, extent_osr,
daymet_osr, extent_cs)
output_extent = drigo.intersect_extents([daymet_extent, output_extent])
output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs)
output_geo = output_extent.geo(daymet_cs)
logging.debug(' Geo: {}'.format(output_geo))
logging.debug(' Extent: {}'.format(output_extent))
else:
output_extent = daymet_extent.copy()
output_geo = daymet_geo[:]
# output_shape = output_extent.shape(cs=daymet_cs)
xi, yi = drigo.array_geo_offsets(daymet_geo, output_geo, daymet_cs)
output_rows, output_cols = output_extent.shape(daymet_cs)
logging.debug(' Shape: {} {}'.format(output_rows, output_cols))
logging.debug(' Offsets: {} {} (x y)'.format(xi, yi))
# Read the elevation array
elev_array = drigo.raster_to_array(elev_raster,
mask_extent=output_extent,
return_nodata=False)
pair_array = refet.calcs._air_pressure_func(elev_array)
del elev_array
# Process each variable
input_var = 'vp'
output_var = 'ea'
logging.info("\nVariable: {}".format(input_var))
# Build output folder
var_ws = os.path.join(output_ws, output_var)
if not os.path.isdir(var_ws):
os.makedirs(var_ws)
# Process each file in the input workspace
for input_name in sorted(os.listdir(netcdf_ws)):
logging.debug("{}".format(input_name))
input_match = daymet_re.match(input_name)
if not input_match:
logging.debug(' Regular expression didn\'t match, skipping')
continue
elif input_match.group('VAR') != input_var:
logging.debug(' Variable didn\'t match, skipping')
continue
year_str = input_match.group('YEAR')
logging.info(" Year: {}".format(year_str))
year_int = int(year_str)
year_days = int(dt.datetime(year_int, 12, 31).strftime('%j'))
if start_dt is not None and year_int < start_dt.year:
logging.debug(' Before start date, skipping')
continue
elif end_dt is not None and year_int > end_dt.year:
logging.debug(' After end date, skipping')
continue
# Build input file path
input_raster = os.path.join(netcdf_ws, input_name)
# if not os.path.isfile(input_raster):
# logging.debug(
# ' Input raster doesn\'t exist, skipping {}'.format(
# input_raster))
# continue
# Build output folder
output_year_ws = os.path.join(var_ws, year_str)
if not os.path.isdir(output_year_ws):
os.makedirs(output_year_ws)
# Read in the DAYMET NetCDF file
input_nc_f = netCDF4.Dataset(input_raster, 'r')
# logging.debug(input_nc_f.variables)
# Check all valid dates in the year
year_dates = _utils.date_range(dt.datetime(year_int, 1, 1),
dt.datetime(year_int + 1, 1, 1))
for date_dt in year_dates:
if start_dt is not None and date_dt < start_dt:
logging.debug(' {} - before start date, skipping'.format(
date_dt.date()))
continue
elif end_dt is not None and date_dt > end_dt:
logging.debug(' {} - after end date, skipping'.format(
date_dt.date()))
continue
else:
logging.info(' {}'.format(date_dt.date()))
output_path = os.path.join(
output_year_ws,
'{}_{}_daymet.img'.format(output_var,
date_dt.strftime('%Y%m%d')))
if os.path.isfile(output_path):
logging.debug(' {}'.format(output_path))
if not overwrite_flag:
logging.debug(' File already exists, skipping')
continue
else:
logging.debug(' File already exists, removing existing')
os.remove(output_path)
doy = int(date_dt.strftime('%j'))
doy_i = range(1, year_days + 1).index(doy)
# Arrays are being read as masked array with a fill value of -9999
# Convert to basic numpy array arrays with nan values
try:
input_ma = input_nc_f.variables[input_var][doy_i,
yi:yi + output_rows,
xi:xi + output_cols]
except IndexError:
logging.info(' date not in netcdf, skipping')
continue
input_nodata = float(input_ma.fill_value)
sph_array = input_ma.data.astype(np.float32)
sph_array[sph_array == input_nodata] = np.nan
# Compute ea [kPa] from specific humidity [kg/kg]
ea_array = (sph_array * pair_array) / (0.622 + 0.378 * sph_array)
# Save the array as 32-bit floats
drigo.array_to_raster(ea_array.astype(np.float32),
output_path,
output_geo=output_geo,
output_proj=daymet_proj,
stats_flag=stats_flag)
del input_ma, ea_array, sph_array
input_nc_f.close()
del input_nc_f
logging.debug('\nScript Complete')
def main(extent_path, output_folder, overwrite_flag=False):
"""Download NED tiles that intersect the study_area
Parameters
----------
extent_path : str
File path to study area shapefile.
output_folder : str
Folder path where files will be saved.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
Notes
-----
Script assumes DEM data is in 1x1 WGS84 degree tiles.
Download 10m (1/3 arc-second) or 30m (1 arc-second) versions from:
10m: rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/13/IMG
30m: rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/1/IMG
For this example, only download 30m DEM.
"""
logging.info('\nDownload NED tiles')
# site_url = 'rockyftp.cr.usgs.gov'
site_url = 'https://prd-tnm.s3.amazonaws.com'
# site_folder = 'vdelivery/Datasets/Staged/Elevation/1/IMG'
site_folder = 'StagedProducts/Elevation/1/IMG'
# This path is what must be queried to list the links
site_file_list_path = 'https://prd-tnm.s3.amazonaws.com/index.html?prefix=StagedProducts/Elevation/1/IMG/'
# Use 1 degree snap point and "cellsize" to get 1x1 degree tiles
tile_osr = drigo.epsg_osr(4326)
tile_x, tile_y, tile_cs = 0, 0, 1
buffer_cells = 0
# Error checking
if not os.path.isfile(extent_path):
logging.error('\nERROR: The input_path does not exist\n')
return False
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
# Check that input is a shapefile
# Get the extent of each feature
logging.debug(' Reading extents')
lat_lon_list = []
shp_driver = ogr.GetDriverByName('ESRI Shapefile')
input_ds = shp_driver.Open(extent_path, 1)
input_osr = drigo.feature_ds_osr(input_ds)
input_layer = input_ds.GetLayer()
input_ftr = input_layer.GetNextFeature()
while input_ftr:
input_geom = input_ftr.GetGeometryRef()
input_extent = drigo.Extent(input_geom.GetEnvelope())
input_extent = input_extent.ogrenv_swap()
input_ftr = input_layer.GetNextFeature()
logging.debug('Input Extent: {}'.format(input_extent))
# Project study area extent to input raster coordinate system
output_extent = drigo.project_extent(input_extent, input_osr, tile_osr)
logging.debug('Output Extent: {}'.format(output_extent))
# Extent needed to select 1x1 degree tiles
tile_extent = output_extent.copy()
tile_extent.adjust_to_snap('EXPAND', tile_x, tile_y, tile_cs)
logging.debug('Tile Extent: {}'.format(tile_extent))
# Get list of avaiable tiles that intersect the extent
lat_lon_list.extend([
(lat, -lon)
for lon in range(int(tile_extent.xmin), int(tile_extent.xmax))
for lat in range(int(tile_extent.ymax), int(tile_extent.ymin), -1)
])
lat_lon_list = sorted(list(set(lat_lon_list)))
# Retrieve a list of files available on the site (keyed by lat/lon)
logging.debug(' Retrieving NED tile list from server')
zip_files = {
m.group(1): x.split('/')[-1]
for x in utils.html_link_list(site_file_list_path)
for m in [re.search('[\w]*(n\d{2}w\d{3})[\w]*.zip', x)] if m
}
# logging.debug(zip_files[:10])
# Attempt to download the tiles
logging.debug('\nDownloading tiles')
logging.info('')
for lat_lon in lat_lon_list:
logging.info('Tile: {}'.format(lat_lon))
lat_lon_key = 'n{:02d}w{:03d}'.format(*lat_lon)
try:
zip_name = zip_files[lat_lon_key]
except KeyError:
logging.exception(
'Error finding zip file for {}, skipping tile'.format(lat_lon))
continue
zip_url = '/'.join([site_url, site_folder, zip_name])
zip_path = os.path.join(output_folder, zip_name)
tile_path = os.path.join(output_folder, '{}.img'.format(lat_lon_key))
logging.debug(' {}'.format(zip_url))
logging.debug(' {}'.format(zip_path))
logging.debug(' {}'.format(tile_path))
if os.path.isfile(tile_path):
if not overwrite_flag:
logging.debug(' tile already exists, skipping')
continue
else:
logging.debug(' tile already exists, removing')
os.remove(tile_path)
utils.url_download(zip_url, zip_path)
logging.debug(' Extracting')
try:
zip_f = zipfile.ZipFile(zip_path)
img_name = [
x for x in zip_f.namelist()
if re.search('[\w]*(n\d{2}w\d{3})[\w]*.img$', x)
][0]
img_path = os.path.join(output_folder, img_name)
zip_f.extract(img_name, output_folder)
zip_f.close()
os.rename(img_path, tile_path)
except Exception as e:
logging.info(' Unhandled exception: {}'.format(e))
try:
os.remove(zip_path)
except Exception as e:
logging.info(' Unhandled exception: {}'.format(e))