def unknown_proj_osr(input_proj):
"""Return the spatial reference object for a projection string"""
try:
output_osr = drigo.epsg_osr(input_proj)
logging.debug(' OSR from EPSG string')
return output_osr
except:
pass
try:
output_osr = drigo.epsg_osr(input_proj.replace('EPSG:'))
logging.debug(' OSR from EPSG integer')
return output_osr
except:
pass
try:
output_osr = drigo.proj_osr(input_proj)
logging.debug(' OSR from WKT')
return output_osr
except:
pass
try:
output_osr = drigo.proj4_osr(input_proj)
logging.debug(' OSR from PROJ4')
return output_osr
except:
pass
try:
output_osr = drigo.raster_path_osr(input_proj)
logging.debug(' OSR from raster path')
return output_osr
except:
pass
try:
output_osr = drigo.feature_path_osr(input_proj)
logging.debug(' OSR from feature path')
return output_osr
except:
pass
return output_osr
def main(start_dt,
end_dt,
input_ws,
output_ws,
remove_ascii_flag=False,
stats_flag=False,
overwrite_flag=False):
"""Extract/convert CIMIS data from ASCII to IMG rasters
Parameters
----------
start_dt : datetime
Start date.
end_dt : datetime
End date.
input_ws : str
Folder path of the input ascii files.
output_ws : str
Folder path of the output IMG rasters.
remove_ascii_flag : bool, optional
If True, remove extracted ascii files (the default is False).
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nExtracting CIMIS data')
logging.debug(' Start date: {}'.format(start_dt))
logging.debug(' End date: {}'.format(end_dt))
# CIMIS rasters to extract
data_list = ['ETo', 'Rso', 'Rs', 'Tdew', 'Tn', 'Tx', 'U2']
# Spatial reference parameters
cimis_proj4 = (
'+proj=aea +lat_1=34 +lat_2=40.5 +lat_0=0 +lon_0=-120 '
'+x_0=0 +y_0=-4000000 +ellps=GRS80 +datum=NAD83 +units=m +no_defs')
cimis_osr = drigo.proj4_osr(cimis_proj4)
# cimis_epsg = 3310 # NAD_1983_California_Teale_Albers
# cimis_osr = drigo.epsg_osr(cimis_epsg)
cimis_osr.MorphToESRI()
cimis_proj = cimis_osr.ExportToWkt()
# Set data types to upper case for comparison
data_list = list(map(lambda x: x.lower(), data_list))
# Look for ascii files
for year_str in sorted(os.listdir(input_ws)):
logging.info('{}'.format(year_str))
if not re.match('^\d{4}$', year_str):
logging.debug(' Not a 4 digit year folder, skipping')
continue
year_ws = os.path.join(input_ws, year_str)
if start_dt is not None and int(year_str) < start_dt.year:
logging.debug(' Before start date, skipping')
continue
elif end_dt is not None and int(year_str) > end_dt.year:
logging.debug(' After end date, skipping')
continue
for date_str in sorted(os.listdir(year_ws)):
date_ws = os.path.join(year_ws, date_str)
try:
date_dt = dt.datetime.strptime(date_str, '%Y_%m_%d')
except ValueError:
logging.debug(
' Invalid folder date format (YYYY_MM_DD), skipping')
continue
if start_dt is not None and date_dt < start_dt:
logging.debug(' Before start date, skipping')
continue
elif end_dt is not None and date_dt > end_dt:
logging.debug(' After end date, skipping')
continue
logging.info(date_str)
for file_item in sorted(os.listdir(date_ws)):
logging.debug(' {}'.format(file_item))
if not file_item.endswith('.asc'):
logging.debug(' Invalid file type (not .asc), skipping')
continue
asc_path = os.path.join(date_ws, file_item)
raster_path = asc_path.replace(input_ws, output_ws)\
.replace('.asc', '.img')
# Only process selected raster types
if file_item.replace('.asc', '').lower() not in data_list:
logging.debug(' Unused file/variable, skipping')
continue
if os.path.isfile(raster_path):
logging.debug(' {}'.format(raster_path))
if not overwrite_flag:
logging.debug(' File already exists, skipping')
continue
else:
logging.debug(
' File already exists, removing existing')
os.remove(raster_path)
# Build the output folder if necessary
if not os.path.isdir(os.path.dirname(raster_path)):
os.makedirs(os.path.dirname(raster_path))
# # Set spatial reference of the ASCII files
# if build_prj_flag:
# output_osr.MorphToESRI()
# cimis_proj = output_osr.ExportToWkt()
# prj_file = open(asc_path.replace('.asc', '.prj'), 'w')
# prj_file.write(cimis_proj)
# prj_file.close()
# Convert the ASCII raster to a IMG raster
drigo.ascii_to_raster(asc_path,
raster_path,
input_type=np.float32,
input_proj=cimis_proj,
stats_flag=stats_flag)
if remove_ascii_flag:
os.remove(asc_path)
# Cleanup
del asc_path, raster_path
logging.debug('\nScript Complete')
def main(ancillary_ws, overwrite_flag=False):
"""Process CIMIS ancillary data
Parameters
----------
ancillary_ws : str
Folder of ancillary rasters.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nProcess CIMIS ancillary data')
# Site URL
site_url = 'http://cimis.casil.ucdavis.edu/cimis/'
# DEM for air pressure calculation
# http://topotools.cr.usgs.gov/gmted_viewer/gmted2010_global_grids.php
elev_full_url = 'http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Grid_ZipFiles/mn30_grd.zip'
elev_full_zip = os.path.join(ancillary_ws, 'mn30_grd.zip')
elev_full_raster = os.path.join(ancillary_ws, 'mn30_grd')
# Get CIMIS grid properties from 2010/01/01 ETo raster
# Grid of the spatial cimis input rasters
# cimis_extent = drigo.Extent((-410000, -660000, 610000, 460000))
# cimis_cs = 2000
# cimis_geo = drigo.extent_geo(cimis_extent, cimis_cs)
# Spatial reference parameters
cimis_proj4 = (
'+proj=aea +lat_1=34 +lat_2=40.5 +lat_0=0 +lon_0=-120 +x_0=0 '
'+y_0=-4000000 +ellps=GRS80 +datum=NAD83 +units=m +no_defs')
cimis_osr = drigo.proj4_osr(cimis_proj4)
# cimis_epsg = 3310 # NAD_1983_California_Teale_Albers
# cimis_osr = drigo.epsg_osr(cimis_epsg)
# Comment this line out if building GeoTIFF instead of IMG
cimis_osr.MorphToESRI()
cimis_proj = cimis_osr.ExportToWkt()
# snap_xmin, snap_ymin = (0, 0)
# Build output workspace if it doesn't exist
if not os.path.isdir(ancillary_ws):
os.makedirs(ancillary_ws)
# File paths
mask_url = site_url + '/2010/01/01/ETo.asc.gz'
# mask_gz = os.path.join(ancillary_ws, 'cimis_mask.asc.gz')
mask_ascii = os.path.join(ancillary_ws, 'cimis_mask.asc')
mask_raster = os.path.join(ancillary_ws, 'cimis_mask.img')
elev_raster = os.path.join(ancillary_ws, 'cimis_elev.img')
lat_raster = os.path.join(ancillary_ws, 'cimis_lat.img')
lon_raster = os.path.join(ancillary_ws, 'cimis_lon.img')
# Download an ETo ASCII raster to generate the mask raster
if overwrite_flag or not os.path.isfile(mask_raster):
logging.info('\nCIMIS mask')
logging.debug(' Downloading')
logging.debug(" {}".format(mask_url))
logging.debug(" {}".format(mask_ascii))
_utils.url_download(mask_url, mask_ascii)
# DEADBEEF - The files do not appeared to be compressed even though
# they have a .asc.gz file extension on the server.
# logging.debug(" {}".format(mask_gz))
# _utils.url_download(mask_url, mask_gz)
#
# they are named .asc.gz
# # Uncompress '.gz' file to a new file
# logging.debug(' Uncompressing')
# logging.debug(' {}'.format(mask_ascii))
# try:
# input_f = gzip.open(mask_gz, 'rb')
# output_f = open(mask_ascii, 'wb')
# output_f.write(input_f.read())
# output_f.close()
# input_f.close()
# del input_f, output_f
# except:
# logging.error(" ERROR EXTRACTING FILE")
# os.remove(mask_gz)
# # Set spatial reference of the ASCII files
# if build_prj_flag:
# prj_file = open(mask_asc.replace('.asc','.prj'), 'w')
# prj_file.write(output_proj)
# prj_file.close()
# Convert the ASCII raster to a IMG raster
logging.debug(' Computing mask')
logging.debug(' {}'.format(mask_raster))
mask_array = drigo.raster_to_array(mask_ascii, return_nodata=False)
cimis_geo = drigo.raster_path_geo(mask_ascii)
# cimis_extent = drigo.raster_path_extent(mask_ascii)
logging.debug(' {}'.format(cimis_geo))
mask_array = np.isfinite(mask_array).astype(np.uint8)
drigo.array_to_raster(mask_array,
mask_raster,
output_geo=cimis_geo,
output_proj=cimis_proj,
output_nodata=0)
# drigo.ascii_to_raster(
# mask_ascii, mask_raster, np.float32, cimis_proj)
os.remove(mask_ascii)
# Compute latitude/longitude rasters
if ((overwrite_flag or not os.path.isfile(lat_raster)
or not os.path.isfile(lat_raster)) and os.path.isfile(mask_raster)):
logging.info('\nCIMIS latitude/longitude')
logging.debug(' {}'.format(lat_raster))
lat_array, lon_array = drigo.raster_lat_lon_func(mask_raster)
drigo.array_to_raster(lat_array,
lat_raster,
output_geo=cimis_geo,
output_proj=cimis_proj)
logging.debug(' {}'.format(lon_raster))
drigo.array_to_raster(lon_array,
lon_raster,
output_geo=cimis_geo,
output_proj=cimis_proj)
# Compute DEM raster
if overwrite_flag or not os.path.isfile(elev_raster):
logging.info('\nCIMIS DEM')
logging.debug(' Downloading GMTED2010 DEM')
logging.debug(" {}".format(elev_full_url))
logging.debug(" {}".format(elev_full_zip))
if overwrite_flag or not os.path.isfile(elev_full_zip):
_utils.url_download(elev_full_url, elev_full_zip)
# Uncompress '.gz' file to a new file
logging.debug(' Uncompressing')
logging.debug(' {}'.format(elev_full_raster))
if overwrite_flag or not os.path.isfile(elev_full_raster):
try:
with zipfile.ZipFile(elev_full_zip, "r") as z:
z.extractall(ancillary_ws)
except:
logging.error(" ERROR EXTRACTING FILE")
os.remove(elev_full_zip)
# Get the extent and cellsize from the mask
logging.debug(' Projecting to CIMIS grid')
cimis_cs = drigo.raster_path_cellsize(mask_raster)[0]
cimis_extent = drigo.raster_path_extent(mask_raster)
logging.debug(' Extent: {}'.format(cimis_extent))
logging.debug(' Cellsize: {}'.format(cimis_cs))
logging.info(' {}'.format(mask_ascii))
if overwrite_flag and os.path.isfile(elev_raster):
subprocess.call(['gdalmanage', 'delete', elev_raster])
if not os.path.isfile(elev_raster):
subprocess.call([
'gdalwarp', '-r', 'average', '-t_srs', cimis_proj4, '-te',
str(cimis_extent.xmin),
str(cimis_extent.ymin),
str(cimis_extent.xmax),
str(cimis_extent.ymax), '-tr',
str(cimis_cs),
str(cimis_cs), '-of', 'HFA', '-co', 'COMPRESSED=TRUE',
elev_full_raster, elev_raster
],
cwd=ancillary_ws)
logging.debug('\nScript Complete')
def main(ancillary_ws=os.getcwd(), zero_elev_nodata_flag=False,
overwrite_flag=False):
"""Process DAYMET ancillary data
Parameters
----------
ancillary_ws : str
Folder of ancillary rasters.
zero_elev_nodata_flag : bool, optional
If True, set elevation nodata values to 0 (the default is False).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nProcess DAYMET ancillary rasters')
# Site URL
# ancillary_url = 'http://daymet.ornl.gov/files/ancillary_files.tgz'
# Build output workspace if it doesn't exist
if not os.path.isdir(ancillary_ws):
os.makedirs(ancillary_ws)
# Input paths
# ancillary_targz = os.path.join(
# ancillary_ws, os.path.basename(ancillary_url))
# dem_nc = os.path.join(ancillary_ws, 'dem_data.nc')
# mask_nc = os.path.join(ancillary_ws, 'mask_data.nc')
# Output paths
dem_raster = os.path.join(ancillary_ws, 'daymet_elev.img')
lat_raster = os.path.join(ancillary_ws, 'daymet_lat.img')
lon_raster = os.path.join(ancillary_ws, 'daymet_lon.img')
# mask_raster = os.path.join(ancillary_ws, 'daymet_mask.img')
# Spatial reference parameters
daymet_proj4 = (
"+proj=lcc +datum=WGS84 +lat_1=25 n "
"+lat_2=60n +lat_0=42.5n +lon_0=100w")
daymet_osr = drigo.proj4_osr(daymet_proj4)
daymet_osr.MorphToESRI()
daymet_proj = daymet_osr.ExportToWkt()
daymet_cs = 1000
# daymet_nodata = -9999
# For now, hardcode the DAYMET extent/geo
snap_xmin, snap_ymin = -4560750, -3090500
daymet_rows, daymet_cols = 8075, 7814
# snap_xmin, snap_ymin = -4659000, -3135000
# daymet_rows, daymet_cols = 8220, 8011
# daymet_geo = (
# snap_xmin, daymet_cs, 0.,
# snap_ymin + daymet_cs * daymet_rows, 0., -daymet_cs)
daymet_extent = drigo.Extent([
snap_xmin, snap_ymin,
snap_xmin + daymet_cs * daymet_cols,
snap_ymin + daymet_cs * daymet_rows])
daymet_geo = daymet_extent.geo(daymet_cs)
logging.debug(" Extent: {}".format(daymet_extent))
logging.debug(" Geo: {}".format(daymet_geo))
# logging.debug(" Cellsize: {}".format(daymet_cs))
# logging.debug(" Shape: {}".format(daymet_extent.shape(daymet_cs)))
# # Download the ancillary raster tar.gz
# if overwrite_flag or not os.path.isfile(ancillary_targz):
# logging.info('\nDownloading ancillary tarball files')
# logging.info(" {}".format(os.path.basename(ancillary_url)))
# logging.debug(" {}".format(ancillary_url))
# logging.debug(" {}".format(ancillary_targz))
# url_download(ancillary_url, ancillary_targz)
# try:
# urllib.urlretrieve(ancillary_url, ancillary_targz)
# except:
# logging.error(" ERROR: {}\n FILE: {}".format(
# sys.exc_info()[0], ancillary_targz))
# os.remove(ancillary_targz)
# # Extract the ancillary rasters
# ancillary_list = [dem_nc]
# # ancillary_list = [dem_nc, mask_nc]
# if (os.path.isfile(ancillary_targz) and
# (overwrite_flag or
# not all([os.path.isfile(os.path.join(ancillary_ws, x))
# for x in ancillary_list]))):
# logging.info('\nExtracting ancillary rasters')
# logging.debug(" {}".format(ancillary_targz))
# tar = tarfile.open(ancillary_targz)
# for member in tar.getmembers():
# print member.name
# member.name = os.path.basename(member.name)
# # Strip off leading numbers from ancillary raster name
# member.name = member.name.split('_', 1)[1]
# member_path = os.path.join(ancillary_ws, member.name)
# if not member.name.endswith('.nc'):
# continue
# elif member_path not in ancillary_list:
# continue
# elif os.path.isfile(member_path):
# continue
# logging.debug(" {}".format(member.name))
# tar.extract(member, ancillary_ws)
# tar.close()
# # Mask
# if ((overwrite_flag or
# not os.path.isfile(mask_raster)) and
# os.path.isfile(mask_nc)):
# logging.info('\nExtracting mask raster')
# mask_nc_f = netCDF4.Dataset(mask_nc, 'r')
# logging.debug(mask_nc_f)
# # logging.debug(mask_nc_f.variables['image'])
# mask_array = mask_nc_f.variables['image'][:]
# mask_array[mask_array == daymet_nodata] = 255
# drigo.array_to_raster(
# mask_array, mask_raster,
# output_geo=daymet_geo, output_proj=daymet_proj,
# output_nodata=255)
# mask_nc_f.close()
# # DEM
# if ((overwrite_flag or not os.path.isfile(dem_raster)) and
# os.path.isfile(dem_nc)):
# logging.info('\nExtracting DEM raster')
# dem_nc_f = netCDF4.Dataset(dem_nc, 'r')
# logging.debug(dem_nc_f)
# # logging.debug(dem_nc_f.variables['image'])
# dem_array = dem_nc_f.variables['image'][:]
# # Rounding issues of the nodata value when converting to float32
# dem_array[dem_array == daymet_nodata] -= 1
# dem_array = dem_array.astype(np.float32)
# if zero_elev_nodata_flag:
# dem_array[dem_array <= daymet_nodata] = 0
# else:
# dem_array[dem_array <= daymet_nodata] = np.nan
# drigo.array_to_raster(
# dem_array, dem_raster,
# output_geo=daymet_geo, output_proj=daymet_proj)
# dem_nc_f.close()
# Latitude/Longitude
if (os.path.isfile(dem_raster) and
(overwrite_flag or
not os.path.isfile(lat_raster) or
not os.path.isfile(lon_raster))):
logging.info('\nDAYMET Latitude/Longitude')
logging.debug(' {}'.format(lat_raster))
lat_array, lon_array = drigo.raster_lat_lon_func(
dem_raster, gcs_cs=0.05)
drigo.array_to_raster(
lat_array.astype(np.float32), lat_raster,
output_geo=daymet_geo, output_proj=daymet_proj)
logging.debug(' {}'.format(lon_raster))
drigo.array_to_raster(
lon_array.astype(np.float32), lon_raster,
output_geo=daymet_geo, output_proj=daymet_proj)
del lat_array, lon_array
logging.debug('\nScript Complete')
def main(input_ws=os.getcwd(),
output_ws=os.getcwd(),
start_date=None,
end_date=None,
remove_gz_flag=False,
remove_ascii_flag=True,
stats_flag=False,
overwrite_flag=False):
"""Extract CIMIS data from tar.gz files
Parameters
----------
input_ws : str
Folder path of the input tar.gz files.
output_ws : str
Folder path of the output IMG rasters.
start_date : str, optional
ISO format date string (YYYY-MM-DD).
end_date : str, optional
ISO format date string (YYYY-MM-DD).
remove_gz_flag : bool, optional
If True, remove downloaded .gz files.
remove_ascii_flag : bool, optional
If True, remove extracted ascii files.
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nExtracting CIMIS data')
# If a date is not set, process 2017
try:
start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d')
logging.debug(' Start date: {}'.format(start_dt))
except:
start_dt = dt.datetime(2017, 1, 1)
logging.info(' Start date: {}'.format(start_dt))
try:
end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d')
logging.debug(' End date: {}'.format(end_dt))
except:
end_dt = dt.datetime(2017, 12, 31)
logging.info(' End date: {}'.format(end_dt))
# CIMIS rasters to extract
data_list = ['ETo', 'Rso', 'Rs', 'Tdew', 'Tn', 'Tx', 'U2']
# Spatial reference parameters
cimis_proj4 = (
"+proj=aea +lat_1=34 +lat_2=40.5 +lat_0=0 +lon_0=-120 " +
"+x_0=0 +y_0=-4000000 +ellps=GRS80 +datum=NAD83 +units=m +no_defs")
cimis_osr = drigo.proj4_osr(cimis_proj4)
# cimis_epsg = 3310 # NAD_1983_California_Teale_Albers
# cimis_osr = drigo.epsg_osr(cimis_epsg)
cimis_osr.MorphToESRI()
cimis_proj = cimis_osr.ExportToWkt()
# Set data types to upper case for comparison
data_list = map(lambda x: x.lower(), data_list)
# Look for .asc.gz files
for year_str in sorted(os.listdir(input_ws)):
logging.info('{}'.format(year_str))
if not re.match('^\d{4}$', year_str):
logging.debug(' Not a 4 digit year folder, skipping')
continue
year_ws = os.path.join(input_ws, year_str)
if start_dt is not None and int(year_str) < start_dt.year:
logging.debug(' Before start date, skipping')
continue
elif end_dt is not None and int(year_str) > end_dt.year:
logging.debug(' After end date, skipping')
continue
for date_str in sorted(os.listdir(year_ws)):
date_ws = os.path.join(year_ws, date_str)
try:
date_dt = dt.datetime.strptime(date_str, '%Y_%m_%d')
except ValueError:
logging.debug(
' Invalid folder date format (YYYY_MM_DD), skipping')
continue
if start_dt is not None and date_dt < start_dt:
logging.debug(' Before start date, skipping')
continue
elif end_dt is not None and date_dt > end_dt:
logging.debug(' After end date, skipping')
continue
logging.info(date_str)
for file_item in sorted(os.listdir(date_ws)):
logging.debug(' {}'.format(file_item))
if not file_item.endswith('.asc.gz'):
logging.debug(
' Invalid file type (not .asc.gz), skipping')
continue
gz_path = os.path.join(date_ws, file_item)
asc_path = gz_path.replace(input_ws,
output_ws).replace('.gz', '')
raster_path = gz_path.replace(input_ws, output_ws).replace(
'.asc.gz', '.img')
# Only process selected raster types
if file_item.replace('.asc.gz', '').lower() not in data_list:
logging.debug(' Unused file/variable, skipping')
continue
if os.path.isfile(raster_path):
logging.debug(' {}'.format(raster_path))
if not overwrite_flag:
logging.debug(' File already exists, skipping')
continue
else:
logging.debug(
' File already exists, removing existing')
os.remove(raster_path)
# Build the output folder if necessary
if not os.path.isdir(os.path.dirname(raster_path)):
os.makedirs(os.path.dirname(raster_path))
# Uncompress '.gz' file to a new file
# DEADBEEF - This needs to catch specific exceptions!
try:
input_f = gzip.open(gz_path, 'rb')
input_data = input_f.read()
input_f.close()
except:
logging.error(" ERROR EXTRACTING FILE")
try:
with open(asc_path, 'wb') as output_f:
output_f.write(input_data)
if remove_gz_flag:
os.remove(gz_path)
except:
logging.error(" ERROR WRITING FILE")
# # Set spatial reference of the ASCII files
# if build_prj_flag:
# output_osr.MorphToESRI()
# cimis_proj = output_osr.ExportToWkt()
# prj_file = open(asc_path.replace('.asc','.prj'), 'w')
# prj_file.write(cimis_proj)
# prj_file.close()
# Convert the ASCII raster to a IMG raster
drigo.ascii_to_raster(asc_path,
raster_path,
input_type=np.float32,
input_proj=cimis_proj,
stats_flag=stats_flag)
if remove_ascii_flag:
os.remove(asc_path)
# Cleanup
del gz_path, asc_path, raster_path
logging.debug('\nScript Complete')