def main(output_folder, overwrite_flag=False):
"""Download soil Available Water Capacity (AWC) raster
Parameters
----------
output_folder : str
Folder path where files will be saved.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
# Composite SSURGO/STATSGO
download_url = 'https://storage.googleapis.com/openet/ssurgo/AWC_WTA_0to10cm_composite.tif'
# STATSGO Only
# download_url = 'https://storage.googleapis.com/openet/statsgo/AWC_WTA_0to10cm_statsgo.tif'
output_name = download_url.split('/')[-1]
output_path = os.path.join(output_folder, output_name)
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
if not os.path.isfile(output_path) or overwrite_flag:
logging.info('\nDownloading AWC')
logging.info(' {}'.format(download_url))
logging.info(' {}'.format(output_path))
_utils.url_download(download_url, output_path)
else:
logging.debug('\nAWC raster already downloaded')
def main(output_folder, overwrite_flag=False):
"""Download Landsat WRS2 descending footprint shapefile
Parameters
----------
output_folder : str
Folder path where files will be saved.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
download_url = ('https://landsat.usgs.gov/sites/default/files/documents/'
'WRS2_descending.zip')
zip_name = 'wrs2_descending.zip'
zip_path = os.path.join(output_folder, zip_name)
output_name = zip_name.replace('.zip', '.shp')
output_path = os.path.join(output_folder, output_name)
# output_path = os.path.join(
# output_folder, os.path.splitext(zip_name)[0], output_name)
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
if ((not os.path.isfile(zip_path) and not os.path.isfile(output_path))
or overwrite_flag):
logging.info('\nDownloading Landsat WRS2 descending shapefile')
logging.info(' {}'.format(download_url))
logging.info(' {}'.format(zip_path))
_utils.url_download(download_url, zip_path)
else:
logging.info('\nFootprint shapefile already downloaded')
if ((overwrite_flag or not os.path.isfile(output_path))
and os.path.isfile(zip_path)):
logging.info('\nExtracting Landsat WRS2 descending shapefile')
logging.debug(' {}'.format(output_path))
with zipfile.ZipFile(zip_path) as zf:
zf.extractall(output_folder)
else:
logging.info('\nFootprint shapefile already extracted')
# If the wrs2_tile_utm_zones.json doesn't exist in the output folder,
# copy it there. Use the script location to figure out the input folder
json_name = 'wrs2_tile_utm_zones.json'
input_folder = os.path.join(os.path.dirname(os.path.dirname(sys.path[0])),
'landsat', 'footprints')
input_json_path = os.path.join(input_folder, 'wrs2_tile_utm_zones.json')
output_json_path = os.path.join(output_folder, 'wrs2_tile_utm_zones.json')
if not os.path.isfile(output_json_path) and os.path.isfile(
input_json_path):
logging.info(
'\nCopying {} to the output footprints folder'.format(json_name))
shutil.copy(input_json_path, output_json_path)
def main(output_folder, year='2011', overwrite_flag=False):
"""Download NLCD raster
Parameters
----------
output_folder : str
Folder path where files will be saved.
year : {2001, 2006, 2011}; optional
NLCD year (the default is 2011).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
download_url = (
'https://prd-tnm.s3.amazonaws.com/StagedProducts/NLCD2011/Land_Cover/'
'CONUS/nlcd_{}_landcover_2011_edition_2014_10_10.zip').format(year)
# download_url = (
# 'http://www.landfire.gov/bulk/downloadfile.php?'
# 'TYPE=nlcd{0}&FNAME=nlcd_{0}_landcover_2011_edition_2014_10_10.zip').format(year)
# download_url = (
# 'http://gisdata.usgs.gov/TDDS/DownloadFile.php?'
# 'TYPE=nlcd{0}&FNAME=nlcd_{0}_landcover_2011_edition_2014_10_10.zip').format(year)
zip_name = 'nlcd_{}_landcover_2011_edition_2014_10_10.zip'.format(year)
zip_path = os.path.join(output_folder, zip_name)
output_name = zip_name.replace('.zip', '.img')
# output_path = os.path.join(output_folder, output_name)
output_path = os.path.join(
output_folder, os.path.splitext(zip_name)[0], output_name)
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
if ((not os.path.isfile(zip_path) and not os.path.isfile(output_path)) or
overwrite_flag):
logging.info('\nDownloading NLCD')
logging.info(' {}'.format(download_url))
logging.info(' {}'.format(zip_path))
_utils.url_download(download_url, zip_path)
else:
logging.info('\nNLCD raster already downloaded')
if ((overwrite_flag or not os.path.isfile(output_path)) and
os.path.isfile(zip_path)):
logging.info('\nExtracting NLCD files')
logging.debug(' {}'.format(output_path))
with zipfile.ZipFile(zip_path) as zf:
zf.extractall(output_folder)
else:
logging.info('\nNLCD raster already extracted')
def main(output_folder, overwrite_flag=False):
"""Download Landsat WRS2 descending footprint shapefile
Parameters
----------
output_folder : str
Folder path where files will be saved.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
download_url = (
'https://landsat.usgs.gov/sites/default/files/documents/wrs2_descending.zip')
zip_name = 'wrs2_descending.zip'
zip_path = os.path.join(output_folder, zip_name)
output_name = zip_name.replace('.zip', '.shp')
output_path = os.path.join(output_folder, output_name)
# output_path = os.path.join(
# output_folder, os.path.splitext(zip_name)[0], output_name)
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
if ((not os.path.isfile(zip_path) and not os.path.isfile(output_path)) or
overwrite_flag):
logging.info('\nDownloading Landsat WRS2 descending shapefile')
logging.info(' {}'.format(download_url))
logging.info(' {}'.format(zip_path))
_utils.url_download(download_url, zip_path)
else:
logging.debug('\nFootprint shapefile already downloaded')
if ((overwrite_flag or not os.path.isfile(output_path)) and
os.path.isfile(zip_path)):
logging.info('\nExtracting Landsat WRS2 descending shapefile')
logging.debug(' {}'.format(output_path))
with zipfile.ZipFile(zip_path) as zf:
zf.extractall(output_folder)
else:
logging.debug('\nFootprint shapefile already extracted')
def main(start_dt, end_dt, netcdf_ws, variables=['all'],
overwrite_flag=False):
"""Download DAYMET netcdf files
Data is currently only available for 1980-2017
Parameters
----------
start_dt : datetime
Start date.
end_dt : datetime
End date.
netcdf_ws : str
Root folder of DAYMET data.
variables : list, optional
DAYMET variables to download ('prcp', 'srad', 'vp', 'tmmn', 'tmmx').
Set as ['all'] to download all available variables.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
Notes
-----
https://thredds.daac.ornl.gov/thredds/catalog/ornldaac/1328/catalog.html
"""
logging.info('\nDownloading DAYMET data')
logging.debug(' Start date: {}'.format(start_dt))
logging.debug(' End date: {}'.format(end_dt))
site_url = 'http://thredds.daac.ornl.gov/thredds/fileServer/ornldaac/1328'
# DAYMET rasters to extract
var_full_list = ['prcp', 'srad', 'vp', 'tmin', 'tmax']
if not variables:
logging.error('\nERROR: variables parameter is empty\n')
sys.exit()
elif type(variables) is not list:
# DEADBEEF - I could try converting comma separated strings to lists?
logging.warning('\nERROR: variables parameter must be a list\n')
sys.exit()
elif 'all' in variables:
logging.error('\nDownloading all variables\n {}'.format(
','.join(var_full_list)))
var_list = var_full_list
elif not set(variables).issubset(set(var_full_list)):
logging.error('\nERROR: variables parameter is invalid\n {}'.format(
variables))
sys.exit()
else:
var_list = variables[:]
# Build output workspace if it doesn't exist
if not os.path.isdir(netcdf_ws):
os.makedirs(netcdf_ws)
# DAYMET data is stored by year
year_list = sorted(list(set([
i_dt.year for i_dt in _utils.date_range(
start_dt, end_dt + dt.timedelta(1))])))
year_list = list(map(lambda x: '{:04d}'.format(x), year_list))
# Set data types to upper case for comparison
var_list = list(map(lambda x: x.lower(), var_list))
# Each sub folder in the main folder has all imagery for 1 day
# The path for each subfolder is the /YYYY/MM/DD
logging.info('')
for year_str in year_list:
logging.info(year_str)
# Process each file in sub folder
for variable in var_list:
file_name = 'daymet_v3_{}_{}_na.nc4'.format(variable, year_str)
file_url = '{}/{}/{}'.format(site_url, year_str, file_name)
save_path = os.path.join(netcdf_ws, file_name)
logging.info(' {}'.format(file_name))
logging.debug(' {}'.format(file_url))
logging.debug(' {}'.format(save_path))
if os.path.isfile(save_path):
if not overwrite_flag:
logging.debug(' File already exists, skipping')
continue
else:
logging.debug(' File already exists, removing existing')
os.remove(save_path)
_utils.url_download(file_url, save_path)
logging.debug('\nScript Complete')
def main(output_folder, version='140', overwrite_flag=False):
"""Download LANDFIRE veg. type
Parameters
----------
output_folder : str
Folder path where files will be saved.
version : {'105', '110', '120', '130', '140'}
LANDFIRE version string (the default is '140').
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
version = str(version).replace('.', '')
base_url = 'http://www.landfire.gov/bulk/downloadfile.php?FNAME='
zip_dict = {
'140': 'US_{0}_mosaic-US_{0}EVT_04252017.zip&TYPE=landfire'.format(version),
'130': 'US_{0}_Mosaic-US_{0}_EVT_04232015.zip&TYPE=landfire'.format(version),
'120': 'US_{0}_Mosaic-US_{0}_EVT_06142017.zip&TYPE=landfire'.format(version),
'110': 'US_{0}_mosaic_Refresh-US_{0}EVT_09122104.zip&TYPE=landfire'.format(version),
'105': 'US_{0}_mosaic_Refresh-US_{0}evt_09122104.zip&TYPE=landfire'.format(version),
}
download_url = base_url + zip_dict[version]
output_name = 'US_{}_EVT'.format(version)
zip_path = os.path.join(output_folder, output_name + '.zip')
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
if not os.path.isfile(zip_path) or overwrite_flag:
logging.info('\nDownloading LANDFIRE vegetation type')
logging.info(' {}'.format(download_url))
logging.info(' {}'.format(zip_path))
_utils.url_download(download_url, zip_path)
else:
logging.debug('\nLANDFIRE raster already downloaded')
if os.path.isfile(zip_path):
logging.info('\nExtracting LANDFIRE files')
with zipfile.ZipFile(zip_path) as zf:
# Extract files using zip naming and folder structure
# zf.extractall(output_folder)
# Ignore top level zip folder name
for member in zf.namelist():
# Replace root folder and switch to OS separator
output_path = list(member.split('/'))
output_path[0] = output_name
output_path = os.sep.join(output_path)
output_ws = os.path.join(
output_folder, os.path.dirname(output_path))
# Skip directories
if not os.path.basename(output_path):
continue
# Only keep "grid" files
if 'Grid' not in output_path:
continue
# Build output directories
if not os.path.isdir(output_ws):
os.makedirs(output_ws)
# Extract
logging.debug(' {}'.format(output_path))
source = zf.open(member)
target = open(os.path.join(output_folder, output_path), "wb")
with source, target:
shutil.copyfileobj(source, target)
else:
logging.debug('\nLANDFIRE raster already extracted')
def main(netcdf_ws=os.getcwd(),
variables=['etr', 'pr'],
start_date=None,
end_date=None,
overwrite_flag=False):
"""Download GRIDMET netcdf files
Parameters
----------
netcdf_ws : str
Folder of GRIDMET netcdf files.
variable : list, optional
GRIDMET variables to download (the default is ['etr', 'ppt']).
Choices: 'eto', 'etr', 'pr', 'srad', 'sph', 'tmmn', 'tmmx', 'vs'
start_date : str, optional
ISO format date (YYYY-MM-DD).
end_date : str, optional
ISO format date (YYYY-MM-DD).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('Downloading GRIDMET data\n')
site_url = 'https://www.northwestknowledge.net/metdata/data'
# If a date is not set, process 2017
try:
start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d')
logging.debug(' Start date: {}'.format(start_dt))
except:
start_dt = dt.datetime(2017, 1, 1)
logging.info(' Start date: {}'.format(start_dt))
try:
end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d')
logging.debug(' End date: {}'.format(end_dt))
except:
end_dt = dt.datetime(2017, 12, 31)
logging.info(' End date: {}'.format(end_dt))
# GRIDMET rasters to extract
data_full_list = ['eto', 'etr', 'pr', 'srad', 'sph', 'tmmn', 'tmmx', 'vs']
if not variables:
logging.error('\nERROR: variables parameter is empty\n')
sys.exit()
elif type(variables) is not list:
# DEADBEEF - I could try converting comma separated strings to lists?
logging.warning('\nERROR: variables parameter must be a list\n')
sys.exit()
# elif 'all' in variables:
# logging.error('Downloading all variables\n {}'.format(
# ','.join(data_full_list)))
# data_list = data_full_list
# elif 'eto' in variables or 'etr' in variables:
# data_etr_list = ['srad', 'sph', 'tmmn', 'tmmx', 'vs']
# logging.error(
# 'Downloading all variables needed to compute ETr/ETo\n {}'.format(
# ','.join(data_etr_list)))
# data_list = data_etr_list
elif not set(variables).issubset(set(data_full_list)):
logging.error(
'\nERROR: variables parameter is invalid\n {}'.format(variables))
sys.exit()
else:
data_list = variables
# Build output workspace if it doesn't exist
if not os.path.isdir(netcdf_ws):
os.makedirs(netcdf_ws)
# GRIDMET data is stored by year
year_list = sorted(
list(
set([
i_dt.year
for i_dt in _utils.date_range(start_dt, end_dt +
dt.timedelta(1))
])))
year_list = map(lambda x: '{:04d}'.format(x), year_list)
# Set data types to upper case for comparison
data_list = list(map(lambda x: x.lower(), data_list))
# Each sub folder in the main folder has all imagery for 1 day
# The path for each subfolder is the /YYYY/MM/DD
logging.info('')
for year_str in year_list:
logging.info(year_str)
# Process each file in sub folder
for data_str in data_list:
file_name = '{}_{}.nc'.format(data_str, year_str)
file_url = '{}/{}'.format(site_url, file_name)
save_path = os.path.join(netcdf_ws, file_name)
logging.info(' {}'.format(file_name))
logging.debug(' {}'.format(file_url))
logging.debug(' {}'.format(save_path))
if os.path.isfile(save_path):
if not overwrite_flag:
logging.debug(' File already exists, skipping')
continue
else:
logging.debug(' File already exists, removing existing')
os.remove(save_path)
_utils.url_download(file_url, save_path)
logging.debug('\nScript Complete')
def main(ancillary_ws=os.getcwd(),
zero_elev_nodata_flag=False,
overwrite_flag=False):
"""Process NLDAS ancillary data
Parameters
----------
ancillary_ws : str
Folder of ancillary rasters.
zero_elev_nodata_flag : bool, optional
If True, set elevation nodata values to 0 (the default is False).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nProcess NLDAS ancillary data')
# Site URLs
mask_url = 'http://ldas.gsfc.nasa.gov/nldas/asc/NLDASmask_UMDunified.asc'
elev_url = 'http://ldas.gsfc.nasa.gov/nldas/asc/gtopomean15k.asc'
nldas_epsg = 'EPSG:4269'
# nldas_epsg = 'EPSG:4326'
nldas_nodata = -9999.0
# Site URLs
# file_re = re.compile(
# 'NLDAS_FORA0125_H.A(?P<YEAR>\d{4})(?P<MONTH>\d{2})(?P<DAY>\d{2}).' +
# '(?P<TIME>\d{4}).002.grb')
# file_re = re.compile(
# 'NLDAS_FORA0125_H.A(?P<DATE>\d{8}).(?P<TIME>\d{4}).002.grb')
# Build output workspace if it doesn't exist
if not os.path.isdir(ancillary_ws):
os.makedirs(ancillary_ws)
# Input paths
input_elev_ascii = os.path.join(ancillary_ws, os.path.basename(elev_url))
input_mask_ascii = os.path.join(ancillary_ws, os.path.basename(mask_url))
# Output paths
elev_ascii = os.path.join(ancillary_ws, 'nldas_elev.asc')
mask_ascii = os.path.join(ancillary_ws, 'nldas_mask.asc')
lat_ascii = os.path.join(ancillary_ws, 'nldas_lat.asc')
lon_ascii = os.path.join(ancillary_ws, 'nldas_lon.asc')
elev_raster = os.path.join(ancillary_ws, 'nldas_elev.img')
mask_raster = os.path.join(ancillary_ws, 'nldas_mask.img')
lat_raster = os.path.join(ancillary_ws, 'nldas_lat.img')
lon_raster = os.path.join(ancillary_ws, 'nldas_lon.img')
# Download the elevation data if necessary
logging.info('\nDownloading ASCII files')
if overwrite_flag or not os.path.isfile(input_elev_ascii):
logging.info(" {}".format(os.path.basename(elev_url)))
logging.debug(" {}".format(elev_url))
logging.debug(" {}".format(input_elev_ascii))
_utils.url_download(elev_url, input_elev_ascii)
# Download the land/water mask if necessary
if overwrite_flag or not os.path.isfile(input_mask_ascii):
logging.info(" {}".format(os.path.basename(mask_url)))
logging.debug(" {}".format(elev_url))
logging.debug(" {}".format(input_elev_ascii))
_utils.url_download(mask_url, input_mask_ascii)
# The XYZ ASCII format is expecting LAT/LON/VALUE
# Export new asc files with just the needed columns for each raster
logging.debug('\nParsing input ASCII files')
logging.debug(' {}'.format(elev_ascii))
elev_df = pd.read_table(input_elev_ascii,
header=None,
sep=r"\s+",
engine='python',
names=['COL', 'ROW', 'LAT', 'LON', 'VALUE'])
elev_df = elev_df.sort_values(['LAT', 'LON'])
if zero_elev_nodata_flag:
elev_df.loc[elev_df['VALUE'] == nldas_nodata, 'VALUE'] = 0
elev_df[['LON', 'LAT', 'VALUE']].to_csv(elev_ascii,
header=None,
index=False)
logging.debug(' {}'.format(input_mask_ascii))
mask_df = pd.read_table(input_mask_ascii,
header=None,
sep=r"\s+",
engine='python',
names=['COL', 'ROW', 'LAT', 'LON', 'VALUE'])
mask_df = mask_df.sort_values(['LAT', 'LON'])
mask_df[['LON', 'LAT', 'VALUE']].to_csv(mask_ascii,
header=None,
index=False)
mask_df[['LON', 'LAT', 'LAT']].to_csv(lat_ascii, header=None, index=False)
mask_df[['LON', 'LAT', 'LON']].to_csv(lon_ascii, header=None, index=False)
# Remove existing rasters if necessary
# -overwrite argument could be passed to gdalwarp instead
if overwrite_flag:
logging.info('\nRemoving existing rasters')
if os.path.isfile(elev_raster):
logging.info(' {}'.format(elev_raster))
subprocess.call(['gdalmanage', 'delete', elev_raster])
if os.path.isfile(mask_raster):
logging.info(' {}'.format(mask_raster))
subprocess.call(['gdalmanage', 'delete', mask_raster])
if os.path.isfile(lat_raster):
logging.info(' {}'.format(lat_raster))
subprocess.call(['gdalmanage', 'delete', lat_raster])
if os.path.isfile(lon_raster):
logging.info(' {}'.format(lon_raster))
subprocess.call(['gdalmanage', 'delete', lon_raster])
# Convert XYZ ascii to raster
logging.info('\nConverting ASCII to raster')
if not os.path.isfile(elev_raster):
logging.info(' {}'.format(elev_ascii))
subprocess.call([
'gdalwarp', '-of', 'HFA', '-t_srs', nldas_epsg, '-co',
'COMPRESSED=TRUE', elev_ascii, elev_raster, '-ot', 'Float32',
'-srcnodata',
str(nldas_nodata), '-dstnodata',
str(drigo.numpy_type_nodata(np.float32))
],
cwd=ancillary_ws)
# subprocess.call(
# ['gdal_translate', '-of', 'HFA', '-a_srs', nldas_epsg,
# '-co', 'COMPRESSED=TRUE', elev_ascii, elev_raster],
# cwd=ancillary_ws)
if not os.path.isfile(mask_raster):
logging.info(' {}'.format(mask_ascii))
subprocess.call([
'gdalwarp', '-of', 'HFA', '-t_srs', nldas_epsg, '-co',
'COMPRESSED=TRUE', mask_ascii, mask_raster
],
cwd=ancillary_ws)
if not os.path.isfile(lat_raster):
logging.info(' {}'.format(lat_ascii))
subprocess.call([
'gdalwarp', '-of', 'HFA', '-t_srs', nldas_epsg, '-co',
'COMPRESSED=TRUE', lat_ascii, lat_raster
],
cwd=ancillary_ws)
if not os.path.isfile(lon_raster):
logging.info(' {}'.format(lon_ascii))
subprocess.call([
'gdalwarp', '-of', 'HFA', '-t_srs', nldas_epsg, '-co',
'COMPRESSED=TRUE', lon_ascii, lon_raster
],
cwd=ancillary_ws)
# Cleanup
os.remove(elev_ascii)
os.remove(mask_ascii)
os.remove(lat_ascii)
os.remove(lon_ascii)
logging.debug('\nScript Complete')
def main(start_dt, end_dt, output_ws, variables=['all'], overwrite_flag=False):
"""Download CIMIS data
Parameters
----------
start_dt : datetime
Start date.
end_dt : datetime
End date.
output_ws : str
Folder path of the output ascii files.
variables : list
Choices: 'ETo', 'Rs', 'Tdew', 'Tn', 'Tx', 'U2', 'all'
'K', 'Rnl', 'Rso' can be downloaded but are not needed.
Set as ['all'] to download all variables.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
Notes
-----
The file on the CIMIS server do not appeared to be compressed even though
they have a .asc.gz file extension.
The files will be saved directly to ASCII type.
"""
logging.info('\nDownloading CIMIS data\n')
logging.debug(' Start date: {}'.format(start_dt))
logging.debug(' End date: {}'.format(end_dt))
# Site URL
site_url = 'http://cimis.casil.ucdavis.edu/cimis/'
# CIMIS rasters to extract
data_full_list = ['ETo', 'Rso', 'Rs', 'Tdew', 'Tn', 'Tx', 'U2']
if not variables:
logging.error('\nERROR: variables parameter is empty\n')
sys.exit()
elif type(variables) is not list:
logging.error('\nERROR: variables parameter must be a list\n')
sys.exit()
elif 'all' in variables:
logging.error('Downloading all variables\n {}'.format(
','.join(data_full_list)))
data_list = ['ETo', 'Rso', 'Rs', 'Tdew', 'Tn', 'Tx', 'U2']
elif not set(variables).issubset(set(data_full_list)):
logging.error(
'\nERROR: variables parameter is invalid\n {}'.format(variables))
sys.exit()
else:
data_list = variables
# Build output workspace if it doesn't exist
if not os.path.isdir(output_ws):
os.makedirs(output_ws)
# Set data types to upper case for comparison
data_list = list(map(lambda x: x.lower(), data_list))
# Each sub folder in the main folder has all imagery for 1 day
# The path for each subfolder is the /YYYY/MM/DD
logging.info('')
for input_date in _utils.date_range(start_dt, end_dt + dt.timedelta(1)):
logging.info('{}'.format(input_date.date()))
date_url = site_url + '/' + input_date.strftime("%Y/%m/%d")
logging.debug(' {}'.format(date_url))
# Download a list of all files in the date sub folder
try:
date_html = requests.get(date_url + '/').text
except:
logging.error(" ERROR: {}".format(date_url))
continue
file_list = sorted(
list(set(re.findall(r'href=[\'"]?([^\'" >]+)', date_html))))
if not file_list:
logging.debug(' Empty file list, skipping date')
continue
# Create a separate folder for each day
year_ws = os.path.join(output_ws, input_date.strftime("%Y"))
if not os.path.isdir(year_ws):
os.mkdir(year_ws)
date_ws = os.path.join(year_ws, input_date.strftime("%Y_%m_%d"))
if not os.path.isdir(date_ws):
os.mkdir(date_ws)
# Process each file in sub folder
for file_name in file_list:
if not file_name.endswith('.asc.gz'):
continue
elif file_name.replace('.asc.gz', '').lower() not in data_list:
continue
file_url = '{}/{}'.format(date_url, file_name)
# DEADBEEF - The file on the CIMIS server do not appeared to be
# compressed even though they have a .asc.gz file extension.
# Saving the files directly to ASCII type.
save_path = os.path.join(date_ws,
file_name.replace('.asc.gz', '.asc'))
# save_path = os.path.join(date_ws, file_name)
logging.info(' {}'.format(os.path.basename(save_path)))
logging.debug(' {}'.format(file_url))
logging.debug(' {}'.format(save_path))
if os.path.isfile(save_path):
if not overwrite_flag:
logging.debug(' File already exists, skipping')
continue
else:
logging.debug(' File already exists, removing existing')
os.remove(save_path)
_utils.url_download(file_url, save_path)
logging.debug('\nScript Complete')
def main(ancillary_ws, overwrite_flag=False):
"""Process CIMIS ancillary data
Parameters
----------
ancillary_ws : str
Folder of ancillary rasters.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nProcess CIMIS ancillary data')
# Site URL
site_url = 'http://cimis.casil.ucdavis.edu/cimis/'
# DEM for air pressure calculation
# http://topotools.cr.usgs.gov/gmted_viewer/gmted2010_global_grids.php
elev_full_url = 'http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Grid_ZipFiles/mn30_grd.zip'
elev_full_zip = os.path.join(ancillary_ws, 'mn30_grd.zip')
elev_full_raster = os.path.join(ancillary_ws, 'mn30_grd')
# Get CIMIS grid properties from 2010/01/01 ETo raster
# Grid of the spatial cimis input rasters
# cimis_extent = drigo.Extent((-410000, -660000, 610000, 460000))
# cimis_cs = 2000
# cimis_geo = drigo.extent_geo(cimis_extent, cimis_cs)
# Spatial reference parameters
cimis_proj4 = (
'+proj=aea +lat_1=34 +lat_2=40.5 +lat_0=0 +lon_0=-120 +x_0=0 '
'+y_0=-4000000 +ellps=GRS80 +datum=NAD83 +units=m +no_defs')
cimis_osr = drigo.proj4_osr(cimis_proj4)
# cimis_epsg = 3310 # NAD_1983_California_Teale_Albers
# cimis_osr = drigo.epsg_osr(cimis_epsg)
# Comment this line out if building GeoTIFF instead of IMG
cimis_osr.MorphToESRI()
cimis_proj = cimis_osr.ExportToWkt()
# snap_xmin, snap_ymin = (0, 0)
# Build output workspace if it doesn't exist
if not os.path.isdir(ancillary_ws):
os.makedirs(ancillary_ws)
# File paths
mask_url = site_url + '/2010/01/01/ETo.asc.gz'
# mask_gz = os.path.join(ancillary_ws, 'cimis_mask.asc.gz')
mask_ascii = os.path.join(ancillary_ws, 'cimis_mask.asc')
mask_raster = os.path.join(ancillary_ws, 'cimis_mask.img')
elev_raster = os.path.join(ancillary_ws, 'cimis_elev.img')
lat_raster = os.path.join(ancillary_ws, 'cimis_lat.img')
lon_raster = os.path.join(ancillary_ws, 'cimis_lon.img')
# Download an ETo ASCII raster to generate the mask raster
if overwrite_flag or not os.path.isfile(mask_raster):
logging.info('\nCIMIS mask')
logging.debug(' Downloading')
logging.debug(" {}".format(mask_url))
logging.debug(" {}".format(mask_ascii))
_utils.url_download(mask_url, mask_ascii)
# DEADBEEF - The files do not appeared to be compressed even though
# they have a .asc.gz file extension on the server.
# logging.debug(" {}".format(mask_gz))
# _utils.url_download(mask_url, mask_gz)
#
# they are named .asc.gz
# # Uncompress '.gz' file to a new file
# logging.debug(' Uncompressing')
# logging.debug(' {}'.format(mask_ascii))
# try:
# input_f = gzip.open(mask_gz, 'rb')
# output_f = open(mask_ascii, 'wb')
# output_f.write(input_f.read())
# output_f.close()
# input_f.close()
# del input_f, output_f
# except:
# logging.error(" ERROR EXTRACTING FILE")
# os.remove(mask_gz)
# # Set spatial reference of the ASCII files
# if build_prj_flag:
# prj_file = open(mask_asc.replace('.asc','.prj'), 'w')
# prj_file.write(output_proj)
# prj_file.close()
# Convert the ASCII raster to a IMG raster
logging.debug(' Computing mask')
logging.debug(' {}'.format(mask_raster))
mask_array = drigo.raster_to_array(mask_ascii, return_nodata=False)
cimis_geo = drigo.raster_path_geo(mask_ascii)
# cimis_extent = drigo.raster_path_extent(mask_ascii)
logging.debug(' {}'.format(cimis_geo))
mask_array = np.isfinite(mask_array).astype(np.uint8)
drigo.array_to_raster(mask_array,
mask_raster,
output_geo=cimis_geo,
output_proj=cimis_proj,
output_nodata=0)
# drigo.ascii_to_raster(
# mask_ascii, mask_raster, np.float32, cimis_proj)
os.remove(mask_ascii)
# Compute latitude/longitude rasters
if ((overwrite_flag or not os.path.isfile(lat_raster)
or not os.path.isfile(lat_raster)) and os.path.isfile(mask_raster)):
logging.info('\nCIMIS latitude/longitude')
logging.debug(' {}'.format(lat_raster))
lat_array, lon_array = drigo.raster_lat_lon_func(mask_raster)
drigo.array_to_raster(lat_array,
lat_raster,
output_geo=cimis_geo,
output_proj=cimis_proj)
logging.debug(' {}'.format(lon_raster))
drigo.array_to_raster(lon_array,
lon_raster,
output_geo=cimis_geo,
output_proj=cimis_proj)
# Compute DEM raster
if overwrite_flag or not os.path.isfile(elev_raster):
logging.info('\nCIMIS DEM')
logging.debug(' Downloading GMTED2010 DEM')
logging.debug(" {}".format(elev_full_url))
logging.debug(" {}".format(elev_full_zip))
if overwrite_flag or not os.path.isfile(elev_full_zip):
_utils.url_download(elev_full_url, elev_full_zip)
# Uncompress '.gz' file to a new file
logging.debug(' Uncompressing')
logging.debug(' {}'.format(elev_full_raster))
if overwrite_flag or not os.path.isfile(elev_full_raster):
try:
with zipfile.ZipFile(elev_full_zip, "r") as z:
z.extractall(ancillary_ws)
except:
logging.error(" ERROR EXTRACTING FILE")
os.remove(elev_full_zip)
# Get the extent and cellsize from the mask
logging.debug(' Projecting to CIMIS grid')
cimis_cs = drigo.raster_path_cellsize(mask_raster)[0]
cimis_extent = drigo.raster_path_extent(mask_raster)
logging.debug(' Extent: {}'.format(cimis_extent))
logging.debug(' Cellsize: {}'.format(cimis_cs))
logging.info(' {}'.format(mask_ascii))
if overwrite_flag and os.path.isfile(elev_raster):
subprocess.call(['gdalmanage', 'delete', elev_raster])
if not os.path.isfile(elev_raster):
subprocess.call([
'gdalwarp', '-r', 'average', '-t_srs', cimis_proj4, '-te',
str(cimis_extent.xmin),
str(cimis_extent.ymin),
str(cimis_extent.xmax),
str(cimis_extent.ymax), '-tr',
str(cimis_cs),
str(cimis_cs), '-of', 'HFA', '-co', 'COMPRESSED=TRUE',
elev_full_raster, elev_raster
],
cwd=ancillary_ws)
logging.debug('\nScript Complete')
def main(ancillary_ws=os.getcwd(),
zero_elev_nodata_flag=False,
overwrite_flag=False):
"""Process GRIDMET ancillary data
Parameters
----------
ancillary_ws : str
Folder of ancillary rasters.
zero_elev_nodata_flag : bool, optional
If True, set elevation nodata values to 0 (the default is False).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nProcess GRIDMET ancillary rasters')
# Site URL
elev_url = 'https://climate.northwestknowledge.net/METDATA/data/metdata_elevationdata.nc'
# Manually define the spatial reference and extent of the GRIDMET data
# This could be read in from a raster
gridmet_osr = osr.SpatialReference()
# Assume GRIDMET data is in WGS84 not NAD83 (need to check with John)
gridmet_osr.ImportFromEPSG(4326)
# gridmet_osr.ImportFromEPSG(4326)
gridmet_proj = drigo.osr_proj(gridmet_osr)
gridmet_cs = 1. / 24 # 0.041666666666666666
gridmet_x = -125 + gridmet_cs * 5
gridmet_y = 49 + gridmet_cs * 10
# gridmet_y = lon_array[0,0] - 0.5 * gridmet_cs
# gridmet_y = lat_array[0,0] + 0.5 * gridmet_cs
# gridmet_rows, gridmet_cols = elev_array.shape
gridmet_geo = (gridmet_x, gridmet_cs, 0., gridmet_y, 0., -gridmet_cs)
# gridmet_extent = drigo.geo_extent(
# gridmet_geo, gridmet_rows, gridmet_cols)
# Keep track of the original/full geo-transform and extent
# gridmet_full_geo = (
# gridmet_x, gridmet_cs, 0., gridmet_y, 0., -gridmet_cs)
# gridmet_full_extent = drigo.geo_extent(
# gridmet_geo, gridmet_rows, gridmet_cols)
logging.debug(' X/Y: {} {}'.format(gridmet_x, gridmet_y))
logging.debug(' Geo: {}'.format(gridmet_geo))
logging.debug(' Cellsize: {}'.format(gridmet_cs))
# Build output workspace if it doesn't exist
if not os.path.isdir(ancillary_ws):
os.makedirs(ancillary_ws)
# Output paths
elev_nc = os.path.join(ancillary_ws, os.path.basename(elev_url))
elev_raster = os.path.join(ancillary_ws, 'gridmet_elev.img')
lat_raster = os.path.join(ancillary_ws, 'gridmet_lat.img')
lon_raster = os.path.join(ancillary_ws, 'gridmet_lon.img')
# Compute DEM raster
if overwrite_flag or not os.path.isfile(elev_raster):
logging.info('\nGRIDMET DEM')
logging.info(' Downloading')
logging.debug(' {}'.format(elev_url))
logging.debug(' {}'.format(elev_nc))
_utils.url_download(elev_url, elev_nc)
# try:
# urllib.urlretrieve(elev_url, elev_nc)
# except:
# logging.error(" ERROR: {}\n FILE: {}".format(
# sys.exc_info()[0], elev_nc))
# # Try to remove the file since it may not have completely downloaded
# os.remove(elev_nc)
logging.info(' Extracting')
logging.debug(' {}'.format(elev_raster))
elev_nc_f = netCDF4.Dataset(elev_nc, 'r')
elev_ma = elev_nc_f.variables['elevation'][0, :, :]
elev_array = elev_ma.data.astype(np.float32)
# elev_nodata = float(elev_ma.fill_value)
elev_array[(elev_array == elev_ma.fill_value) |
(elev_array <= -300)] = np.nan
if zero_elev_nodata_flag:
elev_array[np.isnan(elev_array)] = 0
if np.all(np.isnan(elev_array)):
logging.error(
'\nERROR: The elevation array is all nodata, exiting\n')
sys.exit()
drigo.array_to_raster(elev_array,
elev_raster,
output_geo=gridmet_geo,
output_proj=gridmet_proj)
elev_nc_f.close()
# del elev_nc_f, elev_ma, elev_array, elev_nodata
del elev_nc_f, elev_ma, elev_array
os.remove(elev_nc)
# Compute latitude/longitude rasters
if ((overwrite_flag or not os.path.isfile(lat_raster)
or not os.path.isfile(lat_raster)) and os.path.isfile(elev_raster)):
logging.info('\nGRIDMET Latitude/Longitude')
logging.debug(' {}'.format(lat_raster))
lat_array, lon_array = drigo.raster_lat_lon_func(elev_raster)
# Handle the conversion to radians in the other GRIDMET scripts
# lat_array *= (math.pi / 180)
drigo.array_to_raster(lat_array,
lat_raster,
output_geo=gridmet_geo,
output_proj=gridmet_proj)
logging.debug(' {}'.format(lon_raster))
drigo.array_to_raster(lon_array,
lon_raster,
output_geo=gridmet_geo,
output_proj=gridmet_proj)
del lat_array, lon_array
logging.debug('\nScript Complete')
def main(extent_path, output_folder, overwrite_flag=False):
"""Download NED tiles that intersect the study_area
Parameters
----------
extent_path : str
File path to study area shapefile.
output_folder : str
Folder path where files will be saved.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
Notes
-----
Script assumes DEM data is in 1x1 WGS84 degree tiles.
Download 10m (1/3 arc-second) or 30m (1 arc-second) versions from:
10m: rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/13/IMG
30m: rockyftp.cr.usgs.gov/vdelivery/Datasets/Staged/Elevation/1/IMG
For this example, only download 30m DEM.
"""
logging.info('\nDownload NED tiles')
# site_url = 'rockyftp.cr.usgs.gov'
site_url = 'https://prd-tnm.s3.amazonaws.com'
# site_folder = 'vdelivery/Datasets/Staged/Elevation/1/IMG'
site_folder = 'StagedProducts/Elevation/1/IMG'
# This path is what must be queried to list the links
site_file_list_path = 'https://prd-tnm.s3.amazonaws.com/index.html?prefix=StagedProducts/Elevation/1/IMG/'
# Use 1 degree snap point and "cellsize" to get 1x1 degree tiles
tile_osr = drigo.epsg_osr(4326)
tile_x, tile_y, tile_cs = 0, 0, 1
buffer_cells = 0
# Error checking
if not os.path.isfile(extent_path):
logging.error('\nERROR: The input_path does not exist\n')
return False
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
# Check that input is a shapefile
# Get the extent of each feature
logging.debug(' Reading extents')
lat_lon_list = []
shp_driver = ogr.GetDriverByName('ESRI Shapefile')
input_ds = shp_driver.Open(extent_path, 1)
input_osr = drigo.feature_ds_osr(input_ds)
input_layer = input_ds.GetLayer()
input_ftr = input_layer.GetNextFeature()
while input_ftr:
input_geom = input_ftr.GetGeometryRef()
input_extent = drigo.Extent(input_geom.GetEnvelope())
input_extent = input_extent.ogrenv_swap()
input_ftr = input_layer.GetNextFeature()
logging.debug('Input Extent: {}'.format(input_extent))
# Project study area extent to input raster coordinate system
output_extent = drigo.project_extent(input_extent, input_osr, tile_osr)
logging.debug('Output Extent: {}'.format(output_extent))
# Extent needed to select 1x1 degree tiles
tile_extent = output_extent.copy()
tile_extent.adjust_to_snap('EXPAND', tile_x, tile_y, tile_cs)
logging.debug('Tile Extent: {}'.format(tile_extent))
# Get list of avaiable tiles that intersect the extent
lat_lon_list.extend([
(lat, -lon)
for lon in range(int(tile_extent.xmin), int(tile_extent.xmax))
for lat in range(int(tile_extent.ymax), int(tile_extent.ymin), -1)
])
lat_lon_list = sorted(list(set(lat_lon_list)))
# Retrieve a list of files available on the site (keyed by lat/lon)
logging.debug(' Retrieving NED tile list from server')
zip_files = {
m.group(1): x.split('/')[-1]
for x in utils.html_link_list(site_file_list_path)
for m in [re.search('[\w]*(n\d{2}w\d{3})[\w]*.zip', x)] if m
}
# logging.debug(zip_files[:10])
# Attempt to download the tiles
logging.debug('\nDownloading tiles')
logging.info('')
for lat_lon in lat_lon_list:
logging.info('Tile: {}'.format(lat_lon))
lat_lon_key = 'n{:02d}w{:03d}'.format(*lat_lon)
try:
zip_name = zip_files[lat_lon_key]
except KeyError:
logging.exception(
'Error finding zip file for {}, skipping tile'.format(lat_lon))
continue
zip_url = '/'.join([site_url, site_folder, zip_name])
zip_path = os.path.join(output_folder, zip_name)
tile_path = os.path.join(output_folder, '{}.img'.format(lat_lon_key))
logging.debug(' {}'.format(zip_url))
logging.debug(' {}'.format(zip_path))
logging.debug(' {}'.format(tile_path))
if os.path.isfile(tile_path):
if not overwrite_flag:
logging.debug(' tile already exists, skipping')
continue
else:
logging.debug(' tile already exists, removing')
os.remove(tile_path)
utils.url_download(zip_url, zip_path)
logging.debug(' Extracting')
try:
zip_f = zipfile.ZipFile(zip_path)
img_name = [
x for x in zip_f.namelist()
if re.search('[\w]*(n\d{2}w\d{3})[\w]*.img$', x)
][0]
img_path = os.path.join(output_folder, img_name)
zip_f.extract(img_name, output_folder)
zip_f.close()
os.rename(img_path, tile_path)
except Exception as e:
logging.info(' Unhandled exception: {}'.format(e))
try:
os.remove(zip_path)
except Exception as e:
logging.info(' Unhandled exception: {}'.format(e))
