def merge(ctx, files, output, driver, bounds, res, nodata, force_overwrite,
precision, creation_options):
"""Copy valid pixels from input files to an output file.
All files must have the same number of bands, data type, and
coordinate reference system.
Input files are merged in their listed order using the reverse
painter's algorithm. If the output file exists, its values will be
overwritten by input values.
Geospatial bounds and resolution of a new output file in the
units of the input file coordinate reference system may be provided
and are otherwise taken from the first input file.
Note: --res changed from 2 parameters in 0.25.
\b
--res 0.1 0.1 => --res 0.1 (square)
--res 0.1 0.2 => --res 0.1 --res 0.2 (rectangular)
"""
from rasterio.tools.merge import merge as merge_tool
verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 1
logger = logging.getLogger('rio')
output, files = resolve_inout(files=files, output=output)
if os.path.exists(output) and not force_overwrite:
raise click.ClickException(
"Output exists and won't be overwritten without the "
"`-f` option")
sources = [rasterio.open(f) for f in files]
dest, output_transform = merge_tool(sources,
bounds=bounds,
res=res,
nodata=nodata,
precision=precision)
profile = sources[0].profile
profile.pop('affine')
profile['transform'] = output_transform
profile['height'] = dest.shape[1]
profile['width'] = dest.shape[2]
profile['driver'] = driver
profile.update(**creation_options)
with rasterio.open(output, 'w', **profile) as dst:
dst.write(dest)
def merge(ctx, files, output, driver, bounds, res, nodata, force_overwrite,
precision, creation_options):
"""Copy valid pixels from input files to an output file.
All files must have the same number of bands, data type, and
coordinate reference system.
Input files are merged in their listed order using the reverse
painter's algorithm. If the output file exists, its values will be
overwritten by input values.
Geospatial bounds and resolution of a new output file in the
units of the input file coordinate reference system may be provided
and are otherwise taken from the first input file.
Note: --res changed from 2 parameters in 0.25.
\b
--res 0.1 0.1 => --res 0.1 (square)
--res 0.1 0.2 => --res 0.1 --res 0.2 (rectangular)
"""
from rasterio.tools.merge import merge as merge_tool
verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 1
logger = logging.getLogger('rio')
output, files = resolve_inout(files=files, output=output)
if os.path.exists(output) and not force_overwrite:
raise click.ClickException(
"Output exists and won't be overwritten without the "
"`-f` option")
sources = [rasterio.open(f) for f in files]
dest, output_transform = merge_tool(sources, bounds=bounds, res=res,
nodata=nodata, precision=precision)
profile = sources[0].profile
profile.pop('affine')
profile['transform'] = output_transform
profile['height'] = dest.shape[1]
profile['width'] = dest.shape[2]
profile['driver'] = driver
profile.update(**creation_options)
with rasterio.open(output, 'w', **profile) as dst:
dst.write(dest)
def get_topo_file(lon_ex, lat_ex, region=None):
"""
Returns a path to the DEM file covering the desired extent.
If the file is not present, download it. If the extent covers two or
more files, merge them.
Returns a downloaded SRTM file for [-60S;60N], a Greenland DEM if
possible, and GTOPO elsewhere (hihi)
Parameters
----------
lon_ex: (min_lon, max_lon)
lat_ex: (min_lat, max_lat)
Returns
-------
tuple: (path to the dem file, data source)
"""
# Did the user specify a specific SRTM file?
if ("dem_file" in cfg.PATHS) and os.path.exists(cfg.PATHS["dem_file"]):
return cfg.PATHS["dem_file"], "USER"
# If not, do the job ourselves: download and merge stuffs
topodir = cfg.PATHS["topo_dir"]
# TODO: GIMP is in polar stereographic, not easy to test
# would be possible with a salem grid but this is a bit more expensive
# than jsut asking RGI for the region
if int(region) == 5:
gimp_file = _download_alternate_topo_file("gimpdem_90m.tif")
return gimp_file, "GIMP"
# Some regional files I could gather
# Iceland http://viewfinderpanoramas.org/dem3/ISL.zip
# Svalbard http://viewfinderpanoramas.org/dem3/SVALBARD.zip
# NorthCanada (could be larger - need tiles download)
_exs = ([-25.0, -12.0, 63.0, 67.0], [10.0, 34.0, 76.0, 81.0], [-96.0, -60.0, 76.0, 84.0])
_files = ("iceland.tif", "svalbard.tif", "northcanada.tif")
for _ex, _f in zip(_exs, _files):
if (
(np.min(lon_ex) >= _ex[0])
and (np.max(lon_ex) <= _ex[1])
and (np.min(lat_ex) >= _ex[2])
and (np.max(lat_ex) <= _ex[3])
):
r_file = _download_alternate_topo_file(_f)
return r_file, "REGIO"
if (np.min(lat_ex) < -60.0) or (np.max(lat_ex) > 60.0):
# use ASTER V2 for northern lats
zones, units = aster_zone(lon_ex, lat_ex)
sources = []
for z, u in zip(zones, units):
sf = _download_aster_file(z, u)
if sf is not None:
sources.append(sf)
source_str = "ASTER"
else:
# Use SRTM!
zones = srtm_zone(lon_ex, lat_ex)
sources = []
for z in zones:
sources.append(_download_srtm_file(z))
source_str = "SRTM"
if len(sources) < 1:
raise RuntimeError("No topography file available!")
# for the very last cases a very coarse dataset ?
t_file = os.path.join(topodir, "ETOPO1_Ice_g_geotiff.tif")
assert os.path.exists(t_file)
return t_file, "ETOPO1"
if len(sources) == 1:
return sources[0], source_str
else:
# merge
zone_str = "+".join(zones)
bname = source_str.lower() + "_merged_" + zone_str + ".tif"
merged_file = os.path.join(topodir, source_str.lower(), bname)
if not os.path.exists(merged_file):
# write it
rfiles = [rasterio.open(s) for s in sources]
dest, output_transform = merge_tool(rfiles)
profile = rfiles[0].profile
profile.pop("affine")
profile["transform"] = output_transform
profile["height"] = dest.shape[1]
profile["width"] = dest.shape[2]
profile["driver"] = "GTiff"
with rasterio.open(merged_file, "w", **profile) as dst:
dst.write(dest)
return merged_file, source_str + "_MERGED"
def get_topo_file(lon_ex, lat_ex, region=None):
"""
Returns a path to the DEM file covering the desired extent.
If the file is not present, download it. If the extent covers two or
more files, merge them.
Returns a downloaded SRTM file for [-60S;60N], a Greenland DEM if
possible, and GTOPO elsewhere (hihi)
Parameters
----------
lon_ex: (min_lon, max_lon)
lat_ex: (min_lat, max_lat)
Returns
-------
tuple: (path to the dem file, data source)
"""
# Did the user specify a specific SRTM file?
if ('dem_file' in cfg.PATHS) and os.path.exists(cfg.PATHS['dem_file']):
return cfg.PATHS['dem_file'], 'USER'
# If not, do the job ourselves: download and merge stuffs
topodir = cfg.PATHS['topo_dir']
# TODO: GIMP is in polar stereographic, not easy to test
# would be possible with a salem grid but this is a bit more expensive
# than jsut asking RGI for the region
if int(region) == 5:
gimp_file = os.path.join(cfg.PATHS['topo_dir'], 'gimpdem_90m.tif')
assert os.path.exists(gimp_file)
return gimp_file, 'GIMP'
# Some regional files I could gather
# Iceland http://viewfinderpanoramas.org/dem3/ISL.zip
# Svalbard http://viewfinderpanoramas.org/dem3/SVALBARD.zip
# NorthCanada (could be larger - need tiles download)
_exs = (
[-25., -12., 63., 67.],
[10., 34., 76., 81.],
[-96., -60., 76., 84.]
)
_files = (
'iceland.tif',
'svalbard.tif',
'northcanada.tif',
)
for _ex, _f in zip(_exs, _files):
if (np.min(lon_ex) >= _ex[0]) and (np.max(lon_ex) <= _ex[1]) and \
(np.min(lat_ex) >= _ex[2]) and (np.max(lat_ex) <= _ex[3]):
r_file = os.path.join(cfg.PATHS['topo_dir'], _f)
assert os.path.exists(r_file)
return r_file, 'REGIO'
if (np.min(lat_ex) < -60.) or (np.max(lat_ex) > 60.):
# use ASTER V2 for northern lats
zones, units = aster_zone(lon_ex, lat_ex)
sources = []
for z, u in zip(zones, units):
sf = _download_aster_file(z, u)
if sf is not None:
sources.append(sf)
source_str = 'ASTER'
else:
# Use SRTM!
zones = srtm_zone(lon_ex, lat_ex)
sources = []
for z in zones:
sources.append(_download_srtm_file(z))
source_str = 'SRTM'
if len(sources) < 1:
raise RuntimeError('No topography file available!')
# for the very last cases a very coarse dataset ?
t_file = os.path.join(topodir, 'ETOPO1_Ice_g_geotiff.tif')
assert os.path.exists(t_file)
return t_file, 'ETOPO1'
if len(sources) == 1:
return sources[0], source_str
else:
# merge
zone_str = '+'.join(zones)
bname = source_str.lower() + '_merged_' + zone_str + '.tif'
merged_file = os.path.join(topodir, source_str.lower(),
bname)
if not os.path.exists(merged_file):
# write it
rfiles = [rasterio.open(s) for s in sources]
dest, output_transform = merge_tool(rfiles)
profile = rfiles[0].profile
profile.pop('affine')
profile['transform'] = output_transform
profile['height'] = dest.shape[1]
profile['width'] = dest.shape[2]
profile['driver'] = 'GTiff'
with rasterio.open(merged_file, 'w', **profile) as dst:
dst.write(dest)
return merged_file, source_str + '_MERGED'
def get_topo_file(lon_ex, lat_ex, outdir, rgi_region=None, source=None):
"""
Returns a path to a Digital Elevation Model (DEM) file covering the
desired extent.
If the file is not present, download it. If the extent covers two or
more files, they are automatically merged.
By default, returns a downloaded SRTM file for [-60S;60N], and
a corrected DEM3 from viewfinderpanoramas.org else. However, the data
source can be determined manually with the `source` keyword. If a list
of sources is given, one after the other is checked and in case a DEM
exists for the sources, this DEM is returned.
Parameters
----------
lon_ex : tuple, required
A (min_lon, max_lon) tuple delimitating the requested area longitudes.
lat_ex : tuple, required
A (min_lat, max_lat) tuple delimitating the requested area latitudes.
outdir : str, required
Directory where to store the DEM file.
rgi_region : int, optional
The RGI region number (required for the GIMP DEM).
source : str or list of str, optional
If you want to force the use of a certain DEM source. Available are:
- 'USER' : file set in cfg.PATHS['dem_file']
- 'SRTM' : SRTM v4.1
- 'GIMP' : https://bpcrc.osu.edu/gdg/data/gimpdem
- 'RAMP' : http://nsidc.org/data/docs/daac/nsidc0082_ramp_dem.gd.html
- 'DEM3' : http://viewfinderpanoramas.org/
- 'ASTER' : ASTER data
- 'ETOPO1' : last resort, a very coarse global dataset
Returns
-------
tuple: (path to the DEM file, data source).
"""
# If a list of possible sources is given, process them successively
if source is not None and not isinstance(source, string_types):
# check all user options
for s in source:
demf, source_str = get_topo_file(lon_ex,
lat_ex,
outdir,
rgi_region=rgi_region,
source=s)
if os.path.isfile(demf):
return demf, source_str
# If not, do the job ourselves: download and merge stuff:
# GIMP is in polar stereographic, not easy to test if glacier is on the map
# It would be possible with a salem grid but this is a bit more expensive
# Instead, we are just asking RGI for the region
if source == 'GIMP' or (rgi_region is not None and int(rgi_region) == 5):
source = 'GIMP' if source is None else source
if source == 'GIMP':
raise NotImplementedError('GIMP DEM download under development.')
gimp_file = _download_alternate_topo_file(outdir,
'gimpdem_90m.tif')
return gimp_file, source
# Same for Antarctica
if source == 'RAMP' or (rgi_region is not None and int(rgi_region) == 19):
if np.max(lat_ex) > -60:
# special case for some distant islands
source = 'DEM3' if source is None else source
else:
source = 'RAMP' if source is None else source
if source == 'RAMP':
raise NotImplementedError('RAMP DEM download under development.')
gimp_file = _download_alternate_topo_file(
outdir, 'AntarcticDEM_wgs84.tif')
return gimp_file, source
# Anywhere else on Earth we check for DEM3, ASTER, or SRTM
if (np.min(lat_ex) < -60.) or (np.max(lat_ex) > 60.) \
or source == 'DEM3' or source == 'ASTER':
# default is DEM3
source = 'DEM3' if source is None else source
if source == 'DEM3':
# use corrected viewpanoramas.org DEM
zones = dem3_viewpano_zone(lon_ex, lat_ex)
sources = []
for z in zones:
sources.append(download_dem3_viewpano(z, outdir))
source_str = source
if source == 'ASTER':
raise NotImplementedError('ASTER DEM download under development.')
# use ASTER
zones, units = aster_zone(lon_ex, lat_ex)
sources = []
for z, u in zip(zones, units):
sf = _download_aster_file(outdir, z, u)
if sf is not None:
sources.append(sf)
source_str = source
else:
source = 'SRTM' if source is None else source
if source == 'SRTM':
zones = srtm_zone(lon_ex, lat_ex)
sources = []
for z in zones:
sources.append(download_srtm_file(z, outdir))
source_str = source
# For the very last cases a very coarse dataset ?
if source == 'ETOPO1':
t_file = os.path.join(outdir, 'ETOPO1_Ice_g_geotiff.tif')
assert os.path.exists(t_file)
return t_file, 'ETOPO1'
# filter for None (e.g. oceans)
sources = [s for s in sources if s is not None]
if len(sources) < 1:
raise RuntimeError('No topography file available!')
if len(sources) == 1:
return sources[0], source_str
else:
# merge
zone_str = '+'.join(zones)
bname = source_str.lower() + '_merged_' + zone_str + '.tif'
if len(bname) > 200: # file name way too long
import hashlib
hash_object = hashlib.md5(bname.encode())
bname = hash_object.hexdigest() + '.tif'
merged_file = os.path.join(outdir, source_str.lower(), bname)
if not os.path.exists(merged_file):
# check case where wrong zip file is downloaded from
if all(x is None for x in sources):
raise ValueError('Chosen lat/lon values are not available')
# write it
rfiles = [rasterio.open(s) for s in sources]
dest, output_transform = merge_tool(rfiles)
profile = rfiles[0].profile
if 'affine' in profile:
profile.pop('affine')
profile['transform'] = output_transform
profile['height'] = dest.shape[1]
profile['width'] = dest.shape[2]
profile['driver'] = 'GTiff'
with rasterio.open(merged_file, 'w', **profile) as dst:
dst.write(dest)
return merged_file, source_str + '_MERGED'
def _download_dem3_viewpano_unlocked(zone, outdir):
"""Checks if the srtm data is in the directory and if not, download it.
"""
mkdir(outdir)
ofile = os.path.join(outdir, 'dem3_' + zone + '.zip')
outpath = os.path.join(outdir, zone + '.tif')
# check if TIFF file exists already
if os.path.exists(outpath):
return outpath
# some files have a newer version 'v2'
if zone in [
'R33', 'R34', 'R35', 'R36', 'R37', 'R38', 'Q32', 'Q33', 'Q34',
'Q35', 'Q36', 'Q37', 'Q38', 'Q39', 'Q40', 'P31', 'P32', 'P33',
'P34', 'P35', 'P36', 'P37', 'P38', 'P39', 'P40'
]:
ifile = 'http://viewfinderpanoramas.org/dem3/' + zone + 'v2.zip'
elif zone in ['01-15', '16-30', '31-45', '46-60']:
ifile = 'http://viewfinderpanoramas.org/ANTDEM3/' + zone + '.zip'
else:
ifile = 'http://viewfinderpanoramas.org/dem3/' + zone + '.zip'
if not os.path.exists(ofile):
retry_counter = 0
retry_max = 5
while True:
# Try to download
try:
retry_counter += 1
progress_urlretrieve(ifile, ofile)
with zipfile.ZipFile(ofile) as zf:
zf.extractall(outdir)
break
except HTTPError as err:
# This works well for py3
if err.code == 404:
# Ok so this *should* be an ocean tile
return None
elif (500 <= err.code < 600) and retry_counter <= retry_max:
print("Downloading DEM3 data failed with HTTP error %s, "
"retrying in 10 seconds... %s/%s" %
(err.code, retry_counter, retry_max))
time.sleep(10)
continue
else:
raise
except ContentTooShortError:
print("Downloading DEM3 data failed with ContentTooShortError"
" error %s, retrying in 10 seconds... %s/%s" %
(err.code, retry_counter, retry_max))
time.sleep(10)
continue
except zipfile.BadZipfile:
# This is for py2
# Ok so this *should* be an ocean tile
return None
# Serious issue: sometimes, if a southern hemisphere URL is queried for
# download and there is none, a NH zip file os downloaded.
# Example: http://viewfinderpanoramas.org/dem3/SN29.zip yields N29!
# BUT: There are southern hemisphere files that download properly. However,
# the unzipped folder has the file name of
# the northern hemisphere file. Some checks if correct file exists:
if len(zone) == 4 and zone.startswith('S'):
zonedir = os.path.join(outdir, zone[1:])
else:
zonedir = os.path.join(outdir, zone)
globlist = glob.glob(os.path.join(zonedir, '*.hgt'))
# take care of the special file naming cases
if zone in DEM3REG.keys():
globlist = glob.glob(os.path.join(outdir, '*', '*.hgt'))
if not globlist:
raise RuntimeError("We should have some files here, but we don't")
# merge the single HGT files (can be a bit ineffective, because not every
# single file might be exactly within extent...)
rfiles = [rasterio.open(s) for s in globlist]
dest, output_transform = merge_tool(rfiles)
profile = rfiles[0].profile
if 'affine' in profile:
profile.pop('affine')
profile['transform'] = output_transform
profile['height'] = dest.shape[1]
profile['width'] = dest.shape[2]
profile['driver'] = 'GTiff'
with rasterio.open(outpath, 'w', **profile) as dst:
dst.write(dest)
assert os.path.exists(outpath)
# delete original files to spare disk space (Can cause problems on Windows
# that's why try/except
try:
for s in globlist:
os.remove(s)
except PermissionError:
pass
return outpath
def get_topo_file(lon_ex, lat_ex, region=None):
"""
Returns a path to the DEM file covering the desired extent.
If the file is not present, download it. If the extent covers two or
more files, merge them.
Returns a downloaded SRTM file for [-60S;60N], a Greenland DEM if
possible, and GTOPO elsewhere (hihi)
Parameters
----------
lon_ex: (min_lon, max_lon)
lat_ex: (min_lat, max_lat)
Returns
-------
tuple: (path to the dem file, data source)
"""
# Did the user specify a specific SRTM file?
if ('dem_file' in cfg.PATHS) and os.path.exists(cfg.PATHS['dem_file']):
return cfg.PATHS['dem_file'], 'USER'
# If not, do the job ourselves: download and merge stuffs
topodir = cfg.PATHS['topo_dir']
# TODO: GIMP is in polar stereographic, not easy to test
# would be possible with a salem grid but this is a bit more expensive
# than jsut asking RGI for the region
if int(region) == 5:
gimp_file = os.path.join(cfg.PATHS['topo_dir'], 'gimpdem_90m.tif')
assert os.path.exists(gimp_file)
return gimp_file, 'GIMP'
# Some regional files I could gather
# Iceland http://viewfinderpanoramas.org/dem3/ISL.zip
# Svalbard http://viewfinderpanoramas.org/dem3/SVALBARD.zip
# NorthCanada (could be larger - need tiles download)
_exs = ([-25., -12., 63., 67.], [10., 34., 76.,
81.], [-96., -60., 76., 84.])
_files = (
'iceland.tif',
'svalbard.tif',
'northcanada.tif',
)
for _ex, _f in zip(_exs, _files):
if (np.min(lon_ex) >= _ex[0]) and (np.max(lon_ex) <= _ex[1]) and \
(np.min(lat_ex) >= _ex[2]) and (np.max(lat_ex) <= _ex[3]):
r_file = os.path.join(cfg.PATHS['topo_dir'], _f)
assert os.path.exists(r_file)
return r_file, 'REGIO'
if (np.min(lat_ex) < -60.) or (np.max(lat_ex) > 60.):
# use ASTER V2 for northern lats
zones, units = aster_zone(lon_ex, lat_ex)
sources = []
for z, u in zip(zones, units):
sf = _download_aster_file(z, u)
if sf is not None:
sources.append(sf)
source_str = 'ASTER'
else:
# Use SRTM!
zones = srtm_zone(lon_ex, lat_ex)
sources = []
for z in zones:
sources.append(_download_srtm_file(z))
source_str = 'SRTM'
if len(sources) < 1:
raise RuntimeError('No topography file available!')
# for the very last cases a very coarse dataset ?
t_file = os.path.join(topodir, 'ETOPO1_Ice_g_geotiff.tif')
assert os.path.exists(t_file)
return t_file, 'ETOPO1'
if len(sources) == 1:
return sources[0], source_str
else:
# merge
zone_str = '+'.join(zones)
bname = source_str.lower() + '_merged_' + zone_str + '.tif'
merged_file = os.path.join(topodir, source_str.lower(), bname)
if not os.path.exists(merged_file):
# write it
rfiles = [rasterio.open(s) for s in sources]
dest, output_transform = merge_tool(rfiles)
profile = rfiles[0].profile
profile.pop('affine')
profile['transform'] = output_transform
profile['height'] = dest.shape[1]
profile['width'] = dest.shape[2]
profile['driver'] = 'GTiff'
with rasterio.open(merged_file, 'w', **profile) as dst:
dst.write(dest)
return merged_file, source_str + '_MERGED'