def unpack(self, store, tmp): with store.upload_dir() as target: target_dir = os.path.join(target, self.base_dir) mkdir_p(target_dir) with zipfile.ZipFile(tmp.name, 'r') as zfile: zfile.extract(self.target_name, target_dir)
def _download_local_vrts(d, source_store, input_vrts): """ The input VRTs are stored in the source_store, but GDAL doesn't know about any store other than the filesystem. This function downloads all the files referenced in the VRTs to a local, temporary directory and rewrites the vrts to include the local paths instead of the remote ones. It returns the list of list of rewritten VRT paths. """ vrts = [] for rasters in input_vrts: v = [] for r in rasters: filename = os.path.join(d, r) mkdir_p(os.path.dirname(filename)) source_store.get(r, filename) assert os.path.exists(filename), "Tried to get %r from " \ "store and store it to %r, but that doesn't seem to " \ "have worked." % (r, filename) v.append(filename) if v: vrts.append(v) return vrts
def render(self, tmp_dir): logger = logging.getLogger('tiff') bbox = self._mercator_bbox mid_dir = os.path.join(tmp_dir, self.output_dir, str(self.z), str(self.x)) mkdir_p(mid_dir) tile = self.tile_name() logger.debug("Generating tile %r..." % tile) with self.get_datasource(logger) as dst_ds: dst_srs = dst_ds.GetProjection() dst_gt = dst_ds.GetGeoTransform() dst_x_size = dst_ds.RasterXSize dst_y_size = dst_ds.RasterYSize # TIFF compresses best if we stick to integer pixels, using LZW # and the "2" type predictor. we might be able to keep some bits # of precision with float32 and DISCARD_LSB, but that's only # available in GDAL >= 2.0 tile_file = os.path.join(tmp_dir, self.output_dir, tile + ".tif") outfile = tile_file tif_drv = gdal.GetDriverByName("GTiff") tif_ds = tif_drv.Create(outfile, dst_x_size, dst_y_size, 1, gdal.GDT_Int16, options=[ 'TILED=YES', 'BLOCKXSIZE=256', 'BLOCKYSIZE=256', 'COMPRESS=LZW', 'PREDICTOR=2' ]) tif_ds.SetGeoTransform(dst_gt) tif_ds.SetProjection(dst_srs) tif_ds.GetRasterBand(1).SetNoDataValue(-32768) pixels = dst_ds.GetRasterBand(1).ReadAsArray( 0, 0, dst_x_size, dst_y_size) # transform to integer height, clamping the range numpy.clip(pixels, -32768, 32767, out=pixels) tif_ds.GetRasterBand(1).WriteArray(pixels.astype(numpy.int16)) # explicitly delete the datasources. the Python-GDAL docs suggest that # this is a good idea not only to dispose of memory buffers but also # to ensure that the backing file handles are closed. del tif_ds assert os.path.isfile(tile_file) source_names = [type(s).__name__ for s in self.sources] logger.info("Done generating tile %r from %s" % (tile, ", ".join(source_names)))
def unpack(self, store, tmp): img = self.img_name() with store.upload_dir() as target: target_dir = os.path.join(target, self.base_dir) mkdir_p(target_dir) with tmpdir.tmpdir() as d: with zipfile.ZipFile(tmp.name, 'r') as zfile: zfile.extract(img, d) output_file = os.path.join(target, self.output_file()) mask.negative(os.path.join(d, img), "HFA", output_file)
def render(self, tmp_dir): logger = logging.getLogger("tiff") bbox = self._mercator_bbox mid_dir = os.path.join(tmp_dir, self.output_dir, str(self.z), str(self.x)) mkdir_p(mid_dir) tile = self.tile_name() logger.debug("Generating tile %r..." % tile) with self.get_datasource(logger) as dst_ds: dst_srs = dst_ds.GetProjection() dst_gt = dst_ds.GetGeoTransform() dst_x_size = dst_ds.RasterXSize dst_y_size = dst_ds.RasterYSize # TIFF compresses best if we stick to integer pixels, using LZW # and the "2" type predictor. we might be able to keep some bits # of precision with float32 and DISCARD_LSB, but that's only # available in GDAL >= 2.0 tile_file = os.path.join(tmp_dir, self.output_dir, tile + ".tif") outfile = tile_file tif_drv = gdal.GetDriverByName("GTiff") tif_ds = tif_drv.Create( outfile, dst_x_size, dst_y_size, 1, gdal.GDT_Int16, options=["TILED=YES", "BLOCKXSIZE=256", "BLOCKYSIZE=256", "COMPRESS=LZW", "PREDICTOR=2"], ) tif_ds.SetGeoTransform(dst_gt) tif_ds.SetProjection(dst_srs) tif_ds.GetRasterBand(1).SetNoDataValue(-32768) pixels = dst_ds.GetRasterBand(1).ReadAsArray(0, 0, dst_x_size, dst_y_size) # transform to integer height, clamping the range numpy.clip(pixels, -32768, 32767, out=pixels) tif_ds.GetRasterBand(1).WriteArray(pixels.astype(numpy.int16)) # explicitly delete the datasources. the Python-GDAL docs suggest that # this is a good idea not only to dispose of memory buffers but also # to ensure that the backing file handles are closed. del tif_ds assert os.path.isfile(tile_file) source_names = [type(s).__name__ for s in self.sources] logger.info("Done generating tile %r from %s" % (tile, ", ".join(source_names)))
def get(self, source, dest): if 'ETOPO1' in source or 'gmted' in source: cache_path = os.path.join(self.cache_dir, source) if not os.path.exists(cache_path): mkdir_p(os.path.dirname(cache_path)) self.store.get(source, cache_path) # hard link to dest. this makes it non-portable, but means that # we don't have to worry about whether GDAL supports symbolic # links, and we don't have to worry about deleting files, as they # are reference counted by the OS. link(cache_path, dest) else: self.store.get(source, dest)
def unpack(self, store, tmp): # the file inside the TAR is named like this - we're only interested # in the GeoTIFF file, as it already contains all the information # that we need. tif_file = self._tif_file() shift = GREAT_LAKES[self.lake]['datum'] with tmpdir.tmpdir() as tmp_dir: with tarfile.open(tmp.name, mode='r:gz') as tar: tar.extract(tif_file, tmp_dir) tif_path = os.path.join(tmp_dir, tif_file) assert os.path.exists(tif_path), "Didn't extract TIF" with store.upload_dir() as target: mkdir_p(os.path.join(target, self.base_dir)) output_file = os.path.join(target, self.output_file()) mask.datum_shift(tif_path, 'GTiff', output_file, shift)
def unpack(self, store, data_zip, mask_zip=None): with store.upload_dir() as target: target_dir = os.path.join(target, self.base_dir) mkdir_p(target_dir) # if there's no mask, then just extract the SRTM as-is. if mask_zip is None: self._unpack_hgt(data_zip.name, target_dir) return # otherwise, make a temporary directory to keep the SRTM and # mask in while compositing them. with tmpdir.tmpdir() as d: self._unpack_hgt(data_zip.name, d) mask_name = self.fname.replace(".hgt", ".raw") with zipfile.ZipFile(mask_zip.name, 'r') as zfile: zfile.extract(mask_name, d) mask_file = os.path.join(d, mask_name) # mask off the water using the mask raster raw file output_file = os.path.join(target, self.output_file()) mask.raw(os.path.join(d, self.fname), mask_file, 255, "SRTMHGT", output_file)
def unpack(self, store, tmp): with store.upload_dir() as target: mkdir_p(os.path.join(target, self.base_dir)) output_file = os.path.join(target, self.output_file()) mask.negative(tmp.name, "GTiff", output_file)
def render(self, tmp_dir): logger = logging.getLogger('normal') bbox = self._mercator_bbox mid_dir = os.path.join(tmp_dir, self.output_dir, str(self.z), str(self.x)) mkdir_p(mid_dir) tile = self.tile_name() tile_file = os.path.join(tmp_dir, self.output_dir, tile + ".png") logger.debug("Generating tile %r..." % tile) filter_size = 10 outfile = tile_file dst_bbox = bbox.bounds dst_x_size = 256 dst_y_size = 256 dst_x_res = float(dst_bbox[2] - dst_bbox[0]) / dst_x_size dst_y_res = float(dst_bbox[3] - dst_bbox[1]) / dst_y_size dst_srs = osr.SpatialReference() dst_srs.ImportFromEPSG(3857) # expand bbox & image to generate "bleed" for image filter mid_min_x = dst_bbox[0] - filter_size * dst_x_res mid_min_y = dst_bbox[1] - filter_size * dst_y_res mid_max_x = dst_bbox[2] + filter_size * dst_x_res mid_max_y = dst_bbox[3] + filter_size * dst_y_res filter_top_margin = filter_size filter_bot_margin = filter_size filter_lft_margin = filter_size filter_rgt_margin = filter_size # clip bounding box back to the edges of the world. GDAL can handle # wrapping around the world, but it doesn't give the results that # would be expected. if mid_min_x < -0.5 * mercator.MERCATOR_WORLD_SIZE: filter_lft_margin = 0 mid_min_x = dst_bbox[0] if mid_min_y < -0.5 * mercator.MERCATOR_WORLD_SIZE: filter_bot_margin = 0 mid_min_y = dst_bbox[1] if mid_max_x > 0.5 * mercator.MERCATOR_WORLD_SIZE: filter_rgt_margin = 0 mid_max_x = dst_bbox[2] if mid_max_y > 0.5 * mercator.MERCATOR_WORLD_SIZE: filter_top_margin = 0 mid_max_y = dst_bbox[3] mid_x_size = dst_x_size + filter_lft_margin + filter_rgt_margin mid_y_size = dst_y_size + filter_bot_margin + filter_top_margin mid_bbox = (mid_min_x, mid_min_y, mid_max_x, mid_max_y) mid_drv = gdal.GetDriverByName("MEM") mid_ds = mid_drv.Create('', mid_x_size, mid_y_size, 1, gdal.GDT_Float32) mid_gt = (mid_bbox[0], dst_x_res, 0, mid_bbox[3], 0, -dst_y_res) mid_ds.SetGeoTransform(mid_gt) mid_ds.SetProjection(dst_srs.ExportToWkt()) mid_ds.GetRasterBand(1).SetNoDataValue(mercator.FLT_NODATA) # figure out what the approximate scale of the output image is in # lat/lon coordinates. this is used to select the appropriate filter. ll_bbox = self._latlon_bbox ll_x_res = float(ll_bbox.bounds[2] - ll_bbox.bounds[0]) / dst_x_size ll_y_res = float(ll_bbox.bounds[3] - ll_bbox.bounds[1]) / dst_y_size # calculate the resolution of a pixel in real meters for both x and y. # this will be used to scale the gradient so that it's consistent # across zoom levels. ll_mid_x = 0.5 * (ll_bbox.bounds[2] + ll_bbox.bounds[0]) ll_spc_x = 0.5 * (ll_bbox.bounds[2] - ll_bbox.bounds[0]) / dst_x_size ll_mid_y = 0.5 * (ll_bbox.bounds[3] + ll_bbox.bounds[1]) ll_spc_y = 0.5 * (ll_bbox.bounds[3] - ll_bbox.bounds[1]) / dst_y_size geod = Geodesic.WGS84 # NOTE: in defiance of predictability and regularity, the geod methods # take input as (lat, lon) in that order, rather than (x, y) as would # be sensible. # NOTE: at low zooms, taking the width across the tile starts to break # down, so we take the width across a small portion of the interior of # the tile instead. geodesic_res_x = -1.0 / \ geod.Inverse(ll_mid_y, ll_mid_x - ll_spc_x, ll_mid_y, ll_mid_x + ll_spc_x)['s12'] geodesic_res_y = 1.0 / \ geod.Inverse(ll_mid_y - ll_spc_y, ll_mid_x, ll_mid_y + ll_spc_y, ll_mid_x)['s12'] composite.compose(self, mid_ds, logger, min(ll_x_res, ll_y_res)) pixels = mid_ds.GetRasterBand(1).ReadAsArray(0, 0, mid_x_size, mid_y_size) ygrad, xgrad = numpy.gradient(pixels, 2) img = numpy.dstack((geodesic_res_x * xgrad, geodesic_res_y * ygrad, numpy.ones((mid_y_size, mid_x_size)))) # first, we normalise to unit vectors. this puts each element of img # in the range (-1, 1). the "einsum" stuff is serious black magic, but # what it (should be) saying is "for each i,j in the rows and columns, # the output is the sum of img[i,j,k]*img[i,j,k]" - i.e: the square. norm = numpy.sqrt(numpy.einsum('ijk,ijk->ij', img, img)) # the norm is now the "wrong shape" according to numpy, so we need to # copy the norm value out into RGB components. norm_copy = norm[:, :, numpy.newaxis] # dividing the img by norm_copy should give us RGB components with # values between -1 and 1, but we need values between 0 and 255 for # PNG channels. so we move and scale the values to fit in that range. scaled = (128.0 * (img / norm_copy + 1.0)) # and finally clip it to (0, 255) just in case img = numpy.clip(scaled, 0.0, 255.0) # Create output as a 4-channel RGBA image, each (byte) channel # corresponds to x, y, z, h where x, y and z are the respective # components of the normal, and h is an index into a hypsometric tint # table (see HEIGHT_TABLE). dst_ds = mid_drv.Create('', dst_x_size, dst_y_size, 4, gdal.GDT_Byte) dst_gt = (dst_bbox[0], dst_x_res, 0, dst_bbox[3], 0, -dst_y_res) dst_ds.SetGeoTransform(dst_gt) dst_ds.SetProjection(dst_srs.ExportToWkt()) # apply the height mapping function to get the table index. func = numpy.vectorize(_height_mapping_func) hyps = func(pixels).astype(numpy.uint8) # extract the area without the "bleed" margin. ext = img[filter_top_margin:(filter_top_margin+dst_y_size), \ filter_lft_margin:(filter_lft_margin+dst_x_size)] dst_ds.GetRasterBand(1).WriteArray(ext[...,0].astype(numpy.uint8)) dst_ds.GetRasterBand(2).WriteArray(ext[...,1].astype(numpy.uint8)) dst_ds.GetRasterBand(3).WriteArray(ext[...,2].astype(numpy.uint8)) # add hypsometric tint index as alpha channel dst_ds.GetRasterBand(4).WriteArray( hyps[filter_top_margin:(filter_top_margin+dst_y_size), filter_lft_margin:(filter_lft_margin+dst_x_size)]) png_drv = gdal.GetDriverByName("PNG") png_ds = png_drv.CreateCopy(tile_file, dst_ds) # explicitly delete the datasources. the Python-GDAL docs suggest that # this is a good idea not only to dispose of memory buffers but also # to ensure that the backing file handles are closed. del png_ds del dst_ds del mid_ds assert os.path.isfile(tile_file) source_names = [type(s).__name__ for s in self.sources] logger.info("Done generating tile %r from %s" % (tile, ", ".join(source_names)))
def render(self, tmp_dir): logger = logging.getLogger('terrarium') bbox = self._mercator_bbox mid_dir = os.path.join(tmp_dir, self.output_dir, str(self.z), str(self.x)) mkdir_p(mid_dir) tile = self.tile_name() logger.debug("Generating tile %r..." % tile) with self.get_datasource(logger) as dst_ds: dst_srs = dst_ds.GetProjection() dst_gt = dst_ds.GetGeoTransform() dst_x_size = dst_ds.RasterXSize dst_y_size = dst_ds.RasterYSize # we want the output to be 3-channels R, G, B with: # uheight = height + 32768.0 # R = int(height) / 256 # G = int(height) % 256 # B = int(frac(height) * 256) # Looks like gdal doesn't handle "nodata" across multiple channels, # so we'll use R=0, which corresponds to height < 32,513 which is # lower than any depth on Earth, so we should be okay. mem_drv = gdal.GetDriverByName("MEM") mem_ds = mem_drv.Create('', dst_x_size, dst_y_size, 3, gdal.GDT_Byte) mem_ds.SetGeoTransform(dst_gt) mem_ds.SetProjection(dst_srs) mem_ds.GetRasterBand(1).SetNoDataValue(0) pixels = dst_ds.GetRasterBand(1).ReadAsArray( 0, 0, dst_x_size, dst_y_size) # transform to uheight, clamping the range pixels += 32768.0 numpy.clip(pixels, 0.0, 65535.0, out=pixels) r = (pixels / 256).astype(numpy.uint8) res = mem_ds.GetRasterBand(1).WriteArray(r) assert res == gdal.CPLE_None g = (pixels % 256).astype(numpy.uint8) res = mem_ds.GetRasterBand(2).WriteArray(g) assert res == gdal.CPLE_None b = ((pixels * 256) % 256).astype(numpy.uint8) res = mem_ds.GetRasterBand(3).WriteArray(b) assert res == gdal.CPLE_None png_file = os.path.join(tmp_dir, self.output_dir, tile + ".png") png_drv = gdal.GetDriverByName("PNG") png_ds = png_drv.CreateCopy(png_file, mem_ds) # explicitly delete the datasources. the Python-GDAL docs suggest # that this is a good idea not only to dispose of memory buffers # but also to ensure that the backing file handles are closed. del mem_ds del png_ds assert os.path.isfile(png_file) source_names = [type(s).__name__ for s in self.sources] logger.info("Done generating tile %r from %s" % (tile, ", ".join(source_names)))
def render(self, tmp_dir): logger = logging.getLogger('terrarium') bbox = self._mercator_bbox mid_dir = os.path.join(tmp_dir, self.output_dir, str(self.z), str(self.x)) mkdir_p(mid_dir) tile = self.tile_name() logger.debug("Generating tile %r..." % tile) with self.get_datasource(logger) as dst_ds: dst_srs = dst_ds.GetProjection() dst_gt = dst_ds.GetGeoTransform() dst_x_size = dst_ds.RasterXSize dst_y_size = dst_ds.RasterYSize # we want the output to be 3-channels R, G, B with: # uheight = height + 32768.0 # R = int(height) / 256 # G = int(height) % 256 # B = int(frac(height) * 256) # Looks like gdal doesn't handle "nodata" across multiple channels, # so we'll use R=0, which corresponds to height < 32,513 which is # lower than any depth on Earth, so we should be okay. mem_drv = gdal.GetDriverByName("MEM") mem_ds = mem_drv.Create('', dst_x_size, dst_y_size, 3, gdal.GDT_Byte) mem_ds.SetGeoTransform(dst_gt) mem_ds.SetProjection(dst_srs) mem_ds.GetRasterBand(1).SetNoDataValue(0) pixels = dst_ds.GetRasterBand(1).ReadAsArray(0, 0, dst_x_size, dst_y_size) # transform to uheight, clamping the range pixels += 32768.0 numpy.clip(pixels, 0.0, 65535.0, out=pixels) r = (pixels / 256).astype(numpy.uint8) res = mem_ds.GetRasterBand(1).WriteArray(r) assert res == gdal.CPLE_None g = (pixels % 256).astype(numpy.uint8) res = mem_ds.GetRasterBand(2).WriteArray(g) assert res == gdal.CPLE_None b = ((pixels * 256) % 256).astype(numpy.uint8) res = mem_ds.GetRasterBand(3).WriteArray(b) assert res == gdal.CPLE_None png_file = os.path.join(tmp_dir, self.output_dir, tile + ".png") png_drv = gdal.GetDriverByName("PNG") png_ds = png_drv.CreateCopy(png_file, mem_ds) # explicitly delete the datasources. the Python-GDAL docs suggest # that this is a good idea not only to dispose of memory buffers # but also to ensure that the backing file handles are closed. del mem_ds del png_ds assert os.path.isfile(png_file) source_names = [type(s).__name__ for s in self.sources] logger.info("Done generating tile %r from %s" % (tile, ", ".join(source_names)))