def _download(self, hucstr, force=False):
"""Download the data."""
# check directory structure
os.makedirs(self.name_manager.data_dir(), exist_ok=True)
os.makedirs(self.name_manager.folder_name(hucstr), exist_ok=True)
work_folder = self.name_manager.raw_folder_name(hucstr)
os.makedirs(work_folder, exist_ok=True)
filename = self.name_manager.file_name(hucstr)
if not os.path.exists(filename) or force:
url = self._url(hucstr)
downloadfile = os.path.join(work_folder, url.split("/")[-1])
if not os.path.exists(downloadfile) or force:
logging.debug("Attempting to download source for target '%s'" %
filename)
source_utils.download(url, downloadfile, force)
source_utils.unzip(downloadfile, work_folder)
# hope we can find it?
gdb_files = [
f for f in os.listdir(work_folder) if f.endswith('.gdb')
]
assert (len(gdb_files) == 1)
source_utils.move(os.path.join(work_folder, gdb_files[0]),
filename)
if not os.path.exists(filename):
raise RuntimeError(
"Cannot find or download file for source target '%s'" %
filename)
return filename
def _download(self, hucstr, force=False):
"""Find and download data from a given HUC.
Parameters
----------
hucstr : str
The USGS Hydrologic Unit Code
force : bool, optional
If true, re-download even if a file already exists.
Returns
-------
filename : str
The path to the resulting downloaded dataset.
"""
# check directory structure
os.makedirs(self.name_manager.data_dir(), exist_ok=True)
os.makedirs(self.name_manager.folder_name(hucstr), exist_ok=True)
work_folder = self.name_manager.raw_folder_name(hucstr)
os.makedirs(work_folder, exist_ok=True)
filename = self.name_manager.file_name(hucstr)
if not os.path.exists(filename) or force:
url = self._url(hucstr)
downloadfile = os.path.join(work_folder, url.split("/")[-1])
if not os.path.exists(downloadfile) or force:
logging.debug("Attempting to download source for target '%s'" %
filename)
source_utils.download(url, downloadfile, force)
source_utils.unzip(downloadfile, work_folder)
# hope we can find it?
gdb_files = [
f for f in os.listdir(work_folder) if f.endswith('.gdb')
]
assert (len(gdb_files) == 1)
source_utils.move(os.path.join(work_folder, gdb_files[0]),
filename)
if not os.path.exists(filename):
raise RuntimeError(
"Cannot find or download file for source target '%s'" %
filename)
return filename
def _download(self, variable, layer=None, force=False):
"""Downloads individual files via direct download."""
os.makedirs(self.names.folder_name(), exist_ok=True)
if layer is None:
soillevel = ''
else:
soillevel = f'sl{layer}_'
filename = self.names.file_name(variable=variable, soillevel=soillevel)
# download file
filename_base = self.names.file_name_base(variable=variable,
soillevel=soillevel)
url = self.URL + filename_base
source_utils.download(url, filename, force)
# return raster profile
with rasterio.open(filename, 'r') as fid:
profile = fid.profile
return filename, profile
def _download(self, force=False):
"""Download the files, returning list of filenames."""
# check directory structure
os.makedirs(self.names.data_dir(), exist_ok=True)
work_folder = self.names.raw_folder_name()
os.makedirs(work_folder, exist_ok=True)
filename = self.names.file_name()
logging.debug(' filename: {}'.format(filename))
if not os.path.exists(filename) or force:
try:
url = urls[self.layer_name]
except KeyError:
raise NotImplementedError(
'Not yet implemented (but trivial to add, just ask!): {}'.
format(self.layer_name))
logging.warning(
'Downloading NLCD dataset: {} -- this will take a long time, depending upon internet connection.'
.format(self.layer_name))
downloadfile = os.path.join(work_folder, url.split("/")[-1])
if not os.path.exists(downloadfile) or force:
logging.debug("Attempting to download source for target '%s'" %
filename)
source_utils.download(url, downloadfile)
source_utils.unzip(downloadfile, work_folder)
# hope we can find it?
img_files = [
f for f in os.listdir(work_folder) if f.endswith('.img')
]
assert (len(img_files) == 1)
target = os.path.join(work_folder, img_files[0])
os.rename(target, filename)
os.rename(target[:-3] + 'ige', filename[:-3] + 'ige')
with rasterio.open(filename, 'r') as fid:
profile = fid.profile
return filename, profile
def download(self, bounds, force=False):
"""Download the files, returning list of filenames."""
logging.info("Collecting DEMs to tile bounds: {}".format(bounds))
# check directory structure
os.makedirs(self.names.data_dir(), exist_ok=True)
os.makedirs(self.names.raw_folder_name(), exist_ok=True)
# NOTE: we could get these from the REST API, but I would
# prefer to not REQUIRE an internet connection if the data
# already exists.
# tile the bounds in lat/long 1-degree increments
west = int(np.floor(bounds[0]))
south = int(np.floor(bounds[1]))
east = int(np.ceil(bounds[2]))
north = int(np.ceil(bounds[3]))
# generate the list of files needed
filenames = [
self.names.file_name(j + 1, -i) for j in range(south, north)
for i in range(west, east)
]
logging.info(' Need:')
for fname in filenames:
logging.info(' {}'.format(fname))
filenames_success = []
if (any(not os.path.exists(f) for f in filenames) or force):
request = self.request(bounds)
for r in request['items']:
url = r['downloadURL']
north = int(np.round(r['boundingBox']['maxY']))
west = int(np.round(r['boundingBox']['minX']))
filename = self.names.file_name(north, -west)
if filename not in filenames:
# randomly some extra matches are found?
continue
filenames.remove(filename)
if not os.path.exists(filename) or force:
downloadfilename = url.split("/")[-1]
downloadfile = os.path.join(
self.names.raw_folder_name(north, west),
downloadfilename)
assert (downloadfile.endswith('.ZIP')
or downloadfile.endswith('.zip'))
logging.info(
"Attempting to download source for target '%s'" %
filename)
work_dir = self.names.raw_folder_name(north, west)
if not os.path.exists(downloadfile) or force:
source_utils.download(url, downloadfile, force)
source_utils.unzip(downloadfile, work_dir)
unzip_filename = downloadfilename[0:-4]
# hope we can find it?
img_files = []
if os.path.isdir(os.path.join(work_dir, unzip_filename)):
img_files = [
os.path.join(unzip_filename, f)
for f in os.listdir(
os.path.join(work_dir, unzip_filename))
if f.endswith('.' + self.file_format.lower())
]
if len(img_files) == 0:
img_files = [
os.path.join(unzip_filename, f)
for f in os.listdir(
os.path.join(work_dir, unzip_filename))
if f.endswith('.' + self.file_format.upper())
]
if len(img_files) == 0:
img_files = [
f for f in os.listdir(work_dir)
if f.endswith('.' + self.file_format.lower())
]
if len(img_files) == 0:
img_files = [
f for f in os.listdir(work_dir)
if f.endswith('.' + self.file_format.upper())
]
if len(img_files) == 0:
raise RuntimeError(
"{}: Downloaded and unzipped '{}', but cannot find the img file."
.format(self.name, downloadfile))
else:
logging.debug(" Found '{}'".format(
os.path.join(work_dir, img_files[0])))
source_utils.move(os.path.join(work_dir, img_files[0]),
filename)
if not os.path.exists(filename):
raise RuntimeError(
'{}: Cannot find or download file for source target "{}"'
.format(self.name, filename))
else:
filenames_success.append(filename)
if len(filenames) != 0:
logging.warn(
'Potentially missing tiles in the DEM covering bounds: {}'.
format(bounds))
logging.warn(
'This may be a REST API error, or it may be that some tiles are oceanic and not needed.'
)
logging.warn(
'Continuing, but consider this issue if some elevation data is missing.'
)
logging.warn('Missing Tiles:')
for fname in filenames:
logging.warn(' {}'.format(fname))
else:
filenames_success = filenames
return filenames_success