def unpack(self, store, tmp):
with store.upload_dir() as target:
target_dir = os.path.join(target, self.base_dir)
mkdir_p(target_dir)
with zipfile.ZipFile(tmp.name, 'r') as zfile:
zfile.extract(self.target_name, target_dir)
def unpack(self, store, tmp):
with store.upload_dir() as target:
target_dir = os.path.join(target, self.base_dir)
mkdir_p(target_dir)
with zipfile.ZipFile(tmp.name, 'r') as zfile:
zfile.extract(self.target_name, target_dir)
def _download_local_vrts(d, source_store, input_vrts):
"""
The input VRTs are stored in the source_store, but GDAL doesn't know about
any store other than the filesystem. This function downloads all the files
referenced in the VRTs to a local, temporary directory and rewrites the
vrts to include the local paths instead of the remote ones.
It returns the list of list of rewritten VRT paths.
"""
vrts = []
for rasters in input_vrts:
v = []
for r in rasters:
filename = os.path.join(d, r)
mkdir_p(os.path.dirname(filename))
source_store.get(r, filename)
assert os.path.exists(filename), "Tried to get %r from " \
"store and store it to %r, but that doesn't seem to " \
"have worked." % (r, filename)
v.append(filename)
if v:
vrts.append(v)
return vrts
def render(self, tmp_dir):
logger = logging.getLogger('tiff')
bbox = self._mercator_bbox
mid_dir = os.path.join(tmp_dir, self.output_dir, str(self.z),
str(self.x))
mkdir_p(mid_dir)
tile = self.tile_name()
logger.debug("Generating tile %r..." % tile)
with self.get_datasource(logger) as dst_ds:
dst_srs = dst_ds.GetProjection()
dst_gt = dst_ds.GetGeoTransform()
dst_x_size = dst_ds.RasterXSize
dst_y_size = dst_ds.RasterYSize
# TIFF compresses best if we stick to integer pixels, using LZW
# and the "2" type predictor. we might be able to keep some bits
# of precision with float32 and DISCARD_LSB, but that's only
# available in GDAL >= 2.0
tile_file = os.path.join(tmp_dir, self.output_dir, tile + ".tif")
outfile = tile_file
tif_drv = gdal.GetDriverByName("GTiff")
tif_ds = tif_drv.Create(outfile,
dst_x_size,
dst_y_size,
1,
gdal.GDT_Int16,
options=[
'TILED=YES', 'BLOCKXSIZE=256',
'BLOCKYSIZE=256', 'COMPRESS=LZW',
'PREDICTOR=2'
])
tif_ds.SetGeoTransform(dst_gt)
tif_ds.SetProjection(dst_srs)
tif_ds.GetRasterBand(1).SetNoDataValue(-32768)
pixels = dst_ds.GetRasterBand(1).ReadAsArray(
0, 0, dst_x_size, dst_y_size)
# transform to integer height, clamping the range
numpy.clip(pixels, -32768, 32767, out=pixels)
tif_ds.GetRasterBand(1).WriteArray(pixels.astype(numpy.int16))
# explicitly delete the datasources. the Python-GDAL docs suggest that
# this is a good idea not only to dispose of memory buffers but also
# to ensure that the backing file handles are closed.
del tif_ds
assert os.path.isfile(tile_file)
source_names = [type(s).__name__ for s in self.sources]
logger.info("Done generating tile %r from %s" %
(tile, ", ".join(source_names)))
def unpack(self, store, tmp):
img = self.img_name()
with store.upload_dir() as target:
target_dir = os.path.join(target, self.base_dir)
mkdir_p(target_dir)
with tmpdir.tmpdir() as d:
with zipfile.ZipFile(tmp.name, 'r') as zfile:
zfile.extract(img, d)
output_file = os.path.join(target, self.output_file())
mask.negative(os.path.join(d, img), "HFA", output_file)
def unpack(self, store, tmp):
img = self.img_name()
with store.upload_dir() as target:
target_dir = os.path.join(target, self.base_dir)
mkdir_p(target_dir)
with tmpdir.tmpdir() as d:
with zipfile.ZipFile(tmp.name, 'r') as zfile:
zfile.extract(img, d)
output_file = os.path.join(target, self.output_file())
mask.negative(os.path.join(d, img), "HFA", output_file)
def render(self, tmp_dir):
logger = logging.getLogger("tiff")
bbox = self._mercator_bbox
mid_dir = os.path.join(tmp_dir, self.output_dir, str(self.z), str(self.x))
mkdir_p(mid_dir)
tile = self.tile_name()
logger.debug("Generating tile %r..." % tile)
with self.get_datasource(logger) as dst_ds:
dst_srs = dst_ds.GetProjection()
dst_gt = dst_ds.GetGeoTransform()
dst_x_size = dst_ds.RasterXSize
dst_y_size = dst_ds.RasterYSize
# TIFF compresses best if we stick to integer pixels, using LZW
# and the "2" type predictor. we might be able to keep some bits
# of precision with float32 and DISCARD_LSB, but that's only
# available in GDAL >= 2.0
tile_file = os.path.join(tmp_dir, self.output_dir, tile + ".tif")
outfile = tile_file
tif_drv = gdal.GetDriverByName("GTiff")
tif_ds = tif_drv.Create(
outfile,
dst_x_size,
dst_y_size,
1,
gdal.GDT_Int16,
options=["TILED=YES", "BLOCKXSIZE=256", "BLOCKYSIZE=256", "COMPRESS=LZW", "PREDICTOR=2"],
)
tif_ds.SetGeoTransform(dst_gt)
tif_ds.SetProjection(dst_srs)
tif_ds.GetRasterBand(1).SetNoDataValue(-32768)
pixels = dst_ds.GetRasterBand(1).ReadAsArray(0, 0, dst_x_size, dst_y_size)
# transform to integer height, clamping the range
numpy.clip(pixels, -32768, 32767, out=pixels)
tif_ds.GetRasterBand(1).WriteArray(pixels.astype(numpy.int16))
# explicitly delete the datasources. the Python-GDAL docs suggest that
# this is a good idea not only to dispose of memory buffers but also
# to ensure that the backing file handles are closed.
del tif_ds
assert os.path.isfile(tile_file)
source_names = [type(s).__name__ for s in self.sources]
logger.info("Done generating tile %r from %s" % (tile, ", ".join(source_names)))
def get(self, source, dest):
if 'ETOPO1' in source or 'gmted' in source:
cache_path = os.path.join(self.cache_dir, source)
if not os.path.exists(cache_path):
mkdir_p(os.path.dirname(cache_path))
self.store.get(source, cache_path)
# hard link to dest. this makes it non-portable, but means that
# we don't have to worry about whether GDAL supports symbolic
# links, and we don't have to worry about deleting files, as they
# are reference counted by the OS.
link(cache_path, dest)
else:
self.store.get(source, dest)
def get(self, source, dest):
if 'ETOPO1' in source or 'gmted' in source:
cache_path = os.path.join(self.cache_dir, source)
if not os.path.exists(cache_path):
mkdir_p(os.path.dirname(cache_path))
self.store.get(source, cache_path)
# hard link to dest. this makes it non-portable, but means that
# we don't have to worry about whether GDAL supports symbolic
# links, and we don't have to worry about deleting files, as they
# are reference counted by the OS.
link(cache_path, dest)
else:
self.store.get(source, dest)
def unpack(self, store, tmp):
# the file inside the TAR is named like this - we're only interested
# in the GeoTIFF file, as it already contains all the information
# that we need.
tif_file = self._tif_file()
shift = GREAT_LAKES[self.lake]['datum']
with tmpdir.tmpdir() as tmp_dir:
with tarfile.open(tmp.name, mode='r:gz') as tar:
tar.extract(tif_file, tmp_dir)
tif_path = os.path.join(tmp_dir, tif_file)
assert os.path.exists(tif_path), "Didn't extract TIF"
with store.upload_dir() as target:
mkdir_p(os.path.join(target, self.base_dir))
output_file = os.path.join(target, self.output_file())
mask.datum_shift(tif_path, 'GTiff', output_file, shift)
def unpack(self, store, data_zip, mask_zip=None):
with store.upload_dir() as target:
target_dir = os.path.join(target, self.base_dir)
mkdir_p(target_dir)
# if there's no mask, then just extract the SRTM as-is.
if mask_zip is None:
self._unpack_hgt(data_zip.name, target_dir)
return
# otherwise, make a temporary directory to keep the SRTM and
# mask in while compositing them.
with tmpdir.tmpdir() as d:
self._unpack_hgt(data_zip.name, d)
mask_name = self.fname.replace(".hgt", ".raw")
with zipfile.ZipFile(mask_zip.name, 'r') as zfile:
zfile.extract(mask_name, d)
mask_file = os.path.join(d, mask_name)
# mask off the water using the mask raster raw file
output_file = os.path.join(target, self.output_file())
mask.raw(os.path.join(d, self.fname), mask_file, 255,
"SRTMHGT", output_file)
def unpack(self, store, data_zip, mask_zip=None):
with store.upload_dir() as target:
target_dir = os.path.join(target, self.base_dir)
mkdir_p(target_dir)
# if there's no mask, then just extract the SRTM as-is.
if mask_zip is None:
self._unpack_hgt(data_zip.name, target_dir)
return
# otherwise, make a temporary directory to keep the SRTM and
# mask in while compositing them.
with tmpdir.tmpdir() as d:
self._unpack_hgt(data_zip.name, d)
mask_name = self.fname.replace(".hgt", ".raw")
with zipfile.ZipFile(mask_zip.name, 'r') as zfile:
zfile.extract(mask_name, d)
mask_file = os.path.join(d, mask_name)
# mask off the water using the mask raster raw file
output_file = os.path.join(target, self.output_file())
mask.raw(os.path.join(d, self.fname), mask_file, 255,
"SRTMHGT", output_file)
def unpack(self, store, tmp):
with store.upload_dir() as target:
mkdir_p(os.path.join(target, self.base_dir))
output_file = os.path.join(target, self.output_file())
mask.negative(tmp.name, "GTiff", output_file)
def unpack(self, store, tmp):
with store.upload_dir() as target:
mkdir_p(os.path.join(target, self.base_dir))
output_file = os.path.join(target, self.output_file())
mask.negative(tmp.name, "GTiff", output_file)
def render(self, tmp_dir):
logger = logging.getLogger('normal')
bbox = self._mercator_bbox
mid_dir = os.path.join(tmp_dir, self.output_dir,
str(self.z), str(self.x))
mkdir_p(mid_dir)
tile = self.tile_name()
tile_file = os.path.join(tmp_dir, self.output_dir,
tile + ".png")
logger.debug("Generating tile %r..." % tile)
filter_size = 10
outfile = tile_file
dst_bbox = bbox.bounds
dst_x_size = 256
dst_y_size = 256
dst_x_res = float(dst_bbox[2] - dst_bbox[0]) / dst_x_size
dst_y_res = float(dst_bbox[3] - dst_bbox[1]) / dst_y_size
dst_srs = osr.SpatialReference()
dst_srs.ImportFromEPSG(3857)
# expand bbox & image to generate "bleed" for image filter
mid_min_x = dst_bbox[0] - filter_size * dst_x_res
mid_min_y = dst_bbox[1] - filter_size * dst_y_res
mid_max_x = dst_bbox[2] + filter_size * dst_x_res
mid_max_y = dst_bbox[3] + filter_size * dst_y_res
filter_top_margin = filter_size
filter_bot_margin = filter_size
filter_lft_margin = filter_size
filter_rgt_margin = filter_size
# clip bounding box back to the edges of the world. GDAL can handle
# wrapping around the world, but it doesn't give the results that
# would be expected.
if mid_min_x < -0.5 * mercator.MERCATOR_WORLD_SIZE:
filter_lft_margin = 0
mid_min_x = dst_bbox[0]
if mid_min_y < -0.5 * mercator.MERCATOR_WORLD_SIZE:
filter_bot_margin = 0
mid_min_y = dst_bbox[1]
if mid_max_x > 0.5 * mercator.MERCATOR_WORLD_SIZE:
filter_rgt_margin = 0
mid_max_x = dst_bbox[2]
if mid_max_y > 0.5 * mercator.MERCATOR_WORLD_SIZE:
filter_top_margin = 0
mid_max_y = dst_bbox[3]
mid_x_size = dst_x_size + filter_lft_margin + filter_rgt_margin
mid_y_size = dst_y_size + filter_bot_margin + filter_top_margin
mid_bbox = (mid_min_x, mid_min_y, mid_max_x, mid_max_y)
mid_drv = gdal.GetDriverByName("MEM")
mid_ds = mid_drv.Create('', mid_x_size, mid_y_size, 1, gdal.GDT_Float32)
mid_gt = (mid_bbox[0], dst_x_res, 0,
mid_bbox[3], 0, -dst_y_res)
mid_ds.SetGeoTransform(mid_gt)
mid_ds.SetProjection(dst_srs.ExportToWkt())
mid_ds.GetRasterBand(1).SetNoDataValue(mercator.FLT_NODATA)
# figure out what the approximate scale of the output image is in
# lat/lon coordinates. this is used to select the appropriate filter.
ll_bbox = self._latlon_bbox
ll_x_res = float(ll_bbox.bounds[2] - ll_bbox.bounds[0]) / dst_x_size
ll_y_res = float(ll_bbox.bounds[3] - ll_bbox.bounds[1]) / dst_y_size
# calculate the resolution of a pixel in real meters for both x and y.
# this will be used to scale the gradient so that it's consistent
# across zoom levels.
ll_mid_x = 0.5 * (ll_bbox.bounds[2] + ll_bbox.bounds[0])
ll_spc_x = 0.5 * (ll_bbox.bounds[2] - ll_bbox.bounds[0]) / dst_x_size
ll_mid_y = 0.5 * (ll_bbox.bounds[3] + ll_bbox.bounds[1])
ll_spc_y = 0.5 * (ll_bbox.bounds[3] - ll_bbox.bounds[1]) / dst_y_size
geod = Geodesic.WGS84
# NOTE: in defiance of predictability and regularity, the geod methods
# take input as (lat, lon) in that order, rather than (x, y) as would
# be sensible.
# NOTE: at low zooms, taking the width across the tile starts to break
# down, so we take the width across a small portion of the interior of
# the tile instead.
geodesic_res_x = -1.0 / \
geod.Inverse(ll_mid_y, ll_mid_x - ll_spc_x,
ll_mid_y, ll_mid_x + ll_spc_x)['s12']
geodesic_res_y = 1.0 / \
geod.Inverse(ll_mid_y - ll_spc_y, ll_mid_x,
ll_mid_y + ll_spc_y, ll_mid_x)['s12']
composite.compose(self, mid_ds, logger, min(ll_x_res, ll_y_res))
pixels = mid_ds.GetRasterBand(1).ReadAsArray(0, 0, mid_x_size, mid_y_size)
ygrad, xgrad = numpy.gradient(pixels, 2)
img = numpy.dstack((geodesic_res_x * xgrad, geodesic_res_y * ygrad,
numpy.ones((mid_y_size, mid_x_size))))
# first, we normalise to unit vectors. this puts each element of img
# in the range (-1, 1). the "einsum" stuff is serious black magic, but
# what it (should be) saying is "for each i,j in the rows and columns,
# the output is the sum of img[i,j,k]*img[i,j,k]" - i.e: the square.
norm = numpy.sqrt(numpy.einsum('ijk,ijk->ij', img, img))
# the norm is now the "wrong shape" according to numpy, so we need to
# copy the norm value out into RGB components.
norm_copy = norm[:, :, numpy.newaxis]
# dividing the img by norm_copy should give us RGB components with
# values between -1 and 1, but we need values between 0 and 255 for
# PNG channels. so we move and scale the values to fit in that range.
scaled = (128.0 * (img / norm_copy + 1.0))
# and finally clip it to (0, 255) just in case
img = numpy.clip(scaled, 0.0, 255.0)
# Create output as a 4-channel RGBA image, each (byte) channel
# corresponds to x, y, z, h where x, y and z are the respective
# components of the normal, and h is an index into a hypsometric tint
# table (see HEIGHT_TABLE).
dst_ds = mid_drv.Create('', dst_x_size, dst_y_size, 4, gdal.GDT_Byte)
dst_gt = (dst_bbox[0], dst_x_res, 0,
dst_bbox[3], 0, -dst_y_res)
dst_ds.SetGeoTransform(dst_gt)
dst_ds.SetProjection(dst_srs.ExportToWkt())
# apply the height mapping function to get the table index.
func = numpy.vectorize(_height_mapping_func)
hyps = func(pixels).astype(numpy.uint8)
# extract the area without the "bleed" margin.
ext = img[filter_top_margin:(filter_top_margin+dst_y_size), \
filter_lft_margin:(filter_lft_margin+dst_x_size)]
dst_ds.GetRasterBand(1).WriteArray(ext[...,0].astype(numpy.uint8))
dst_ds.GetRasterBand(2).WriteArray(ext[...,1].astype(numpy.uint8))
dst_ds.GetRasterBand(3).WriteArray(ext[...,2].astype(numpy.uint8))
# add hypsometric tint index as alpha channel
dst_ds.GetRasterBand(4).WriteArray(
hyps[filter_top_margin:(filter_top_margin+dst_y_size),
filter_lft_margin:(filter_lft_margin+dst_x_size)])
png_drv = gdal.GetDriverByName("PNG")
png_ds = png_drv.CreateCopy(tile_file, dst_ds)
# explicitly delete the datasources. the Python-GDAL docs suggest that
# this is a good idea not only to dispose of memory buffers but also
# to ensure that the backing file handles are closed.
del png_ds
del dst_ds
del mid_ds
assert os.path.isfile(tile_file)
source_names = [type(s).__name__ for s in self.sources]
logger.info("Done generating tile %r from %s"
% (tile, ", ".join(source_names)))
def render(self, tmp_dir):
logger = logging.getLogger('terrarium')
bbox = self._mercator_bbox
mid_dir = os.path.join(tmp_dir, self.output_dir, str(self.z),
str(self.x))
mkdir_p(mid_dir)
tile = self.tile_name()
logger.debug("Generating tile %r..." % tile)
with self.get_datasource(logger) as dst_ds:
dst_srs = dst_ds.GetProjection()
dst_gt = dst_ds.GetGeoTransform()
dst_x_size = dst_ds.RasterXSize
dst_y_size = dst_ds.RasterYSize
# we want the output to be 3-channels R, G, B with:
# uheight = height + 32768.0
# R = int(height) / 256
# G = int(height) % 256
# B = int(frac(height) * 256)
# Looks like gdal doesn't handle "nodata" across multiple channels,
# so we'll use R=0, which corresponds to height < 32,513 which is
# lower than any depth on Earth, so we should be okay.
mem_drv = gdal.GetDriverByName("MEM")
mem_ds = mem_drv.Create('', dst_x_size, dst_y_size, 3,
gdal.GDT_Byte)
mem_ds.SetGeoTransform(dst_gt)
mem_ds.SetProjection(dst_srs)
mem_ds.GetRasterBand(1).SetNoDataValue(0)
pixels = dst_ds.GetRasterBand(1).ReadAsArray(
0, 0, dst_x_size, dst_y_size)
# transform to uheight, clamping the range
pixels += 32768.0
numpy.clip(pixels, 0.0, 65535.0, out=pixels)
r = (pixels / 256).astype(numpy.uint8)
res = mem_ds.GetRasterBand(1).WriteArray(r)
assert res == gdal.CPLE_None
g = (pixels % 256).astype(numpy.uint8)
res = mem_ds.GetRasterBand(2).WriteArray(g)
assert res == gdal.CPLE_None
b = ((pixels * 256) % 256).astype(numpy.uint8)
res = mem_ds.GetRasterBand(3).WriteArray(b)
assert res == gdal.CPLE_None
png_file = os.path.join(tmp_dir, self.output_dir, tile + ".png")
png_drv = gdal.GetDriverByName("PNG")
png_ds = png_drv.CreateCopy(png_file, mem_ds)
# explicitly delete the datasources. the Python-GDAL docs suggest
# that this is a good idea not only to dispose of memory buffers
# but also to ensure that the backing file handles are closed.
del mem_ds
del png_ds
assert os.path.isfile(png_file)
source_names = [type(s).__name__ for s in self.sources]
logger.info("Done generating tile %r from %s" %
(tile, ", ".join(source_names)))
def render(self, tmp_dir):
logger = logging.getLogger('terrarium')
bbox = self._mercator_bbox
mid_dir = os.path.join(tmp_dir, self.output_dir,
str(self.z), str(self.x))
mkdir_p(mid_dir)
tile = self.tile_name()
logger.debug("Generating tile %r..." % tile)
with self.get_datasource(logger) as dst_ds:
dst_srs = dst_ds.GetProjection()
dst_gt = dst_ds.GetGeoTransform()
dst_x_size = dst_ds.RasterXSize
dst_y_size = dst_ds.RasterYSize
# we want the output to be 3-channels R, G, B with:
# uheight = height + 32768.0
# R = int(height) / 256
# G = int(height) % 256
# B = int(frac(height) * 256)
# Looks like gdal doesn't handle "nodata" across multiple channels,
# so we'll use R=0, which corresponds to height < 32,513 which is
# lower than any depth on Earth, so we should be okay.
mem_drv = gdal.GetDriverByName("MEM")
mem_ds = mem_drv.Create('', dst_x_size, dst_y_size, 3, gdal.GDT_Byte)
mem_ds.SetGeoTransform(dst_gt)
mem_ds.SetProjection(dst_srs)
mem_ds.GetRasterBand(1).SetNoDataValue(0)
pixels = dst_ds.GetRasterBand(1).ReadAsArray(0, 0, dst_x_size, dst_y_size)
# transform to uheight, clamping the range
pixels += 32768.0
numpy.clip(pixels, 0.0, 65535.0, out=pixels)
r = (pixels / 256).astype(numpy.uint8)
res = mem_ds.GetRasterBand(1).WriteArray(r)
assert res == gdal.CPLE_None
g = (pixels % 256).astype(numpy.uint8)
res = mem_ds.GetRasterBand(2).WriteArray(g)
assert res == gdal.CPLE_None
b = ((pixels * 256) % 256).astype(numpy.uint8)
res = mem_ds.GetRasterBand(3).WriteArray(b)
assert res == gdal.CPLE_None
png_file = os.path.join(tmp_dir, self.output_dir,
tile + ".png")
png_drv = gdal.GetDriverByName("PNG")
png_ds = png_drv.CreateCopy(png_file, mem_ds)
# explicitly delete the datasources. the Python-GDAL docs suggest
# that this is a good idea not only to dispose of memory buffers
# but also to ensure that the backing file handles are closed.
del mem_ds
del png_ds
assert os.path.isfile(png_file)
source_names = [type(s).__name__ for s in self.sources]
logger.info("Done generating tile %r from %s"
% (tile, ", ".join(source_names)))
