def tile(self, z, x, y): logger.debug(_("Request WMS tile %s") % ((z, x, y),)) proj = GoogleProjection(self.tilesize, [z]) bbox = proj.tile_bbox((z, x, y)) bbox = proj.project(bbox[:2]) + proj.project(bbox[2:]) bbox = ','.join(map(str, bbox)) # Build WMS request URL encodedparams = urllib.urlencode(self.wmsParams) url = "%s?%s" % (self.url, encodedparams) url += "&bbox=%s" % bbox # commas are not encoded r = DOWNLOAD_RETRIES sleeptime = 1 while r > 0: try: logger.debug(_("Download '%s'") % url) request = urllib2.Request(url) for header, value in self.headers.items(): request.add_header(header, value) f = urllib2.urlopen(request) header = f.info().typeheader assert header == self.wmsParams['format'], "Invalid WMS response type : %s" % header return f.read() except (AssertionError, IOError, httplib.HTTPException), e: logger.debug(_("Download error, retry (%s left). (%s)") % (r, e)) r -= 1 time.sleep(sleeptime) # progressivly sleep longer to wait for this tile if (sleeptime <= 10) and (r % 2 == 0): sleeptime += 1 # increase wait
def find_coverage(self, zoom): """ Returns the bounding box (minx, miny, maxx, maxy) of an adjacent group of tiles at this zoom level. """ # Find a group of adjacent available tiles at this zoom level rows = self._query( """SELECT tile_column, tile_row FROM tiles WHERE zoom_level=? ORDER BY tile_column, tile_row;""", (zoom,), ) t = rows.fetchone() xmin, ymin = t previous = t while t and t[0] - previous[0] <= 1: # adjacent, go on previous = t t = rows.fetchone() xmax, ymax = previous # Transform (xmin, ymin) (xmax, ymax) to pixels S = self.tilesize bottomleft = (xmin * S, (ymax + 1) * S) topright = ((xmax + 1) * S, ymin * S) # Convert center to (lon, lat) proj = GoogleProjection(S, [zoom]) # WGS84 return proj.unproject_pixels(bottomleft, zoom) + proj.unproject_pixels(topright, zoom)
def tile(self, z, x, y): """ Render the specified tile with Mapnik """ logger.debug(_("Render tile %s") % ((z, x, y),)) proj = GoogleProjection(self.tilesize, [z]) return self.render(proj.tile_bbox((z, x, y)))
def grid(self, z, x, y, fields, layer): """ Render the specified grid with Mapnik """ logger.debug(_("Render grid %s") % ((z, x, y), )) proj = GoogleProjection(self.tilesize, [z]) return self.render_grid(proj.tile_bbox((z, x, y)), fields, layer)
def tile(self, z, x, y): """ Render the specified tile with Mapnik """ logger.debug(_("Render tile %s") % ((z, x, y), )) proj = GoogleProjection(self.tilesize, [z]) return self.render(proj.tile_bbox((z, x, y)))
def find_coverage(self, zoom): """ Returns the bounding box (minx, miny, maxx, maxy) of an adjacent group of tiles at this zoom level. """ # Find a group of adjacent available tiles at this zoom level rows = self._query( '''SELECT tile_column, tile_row FROM tiles WHERE zoom_level=? ORDER BY tile_column, tile_row;''', (zoom, )) t = rows.fetchone() xmin, ymin = t previous = t while t and t[0] - previous[0] <= 1: # adjacent, go on previous = t t = rows.fetchone() xmax, ymax = previous # Transform (xmin, ymin) (xmax, ymax) to pixels S = self.tilesize bottomleft = (xmin * S, (ymax + 1) * S) topright = ((xmax + 1) * S, ymin * S) # Convert center to (lon, lat) proj = GoogleProjection(S, [zoom]) # WGS84 return proj.unproject_pixels(bottomleft, zoom) + proj.unproject_pixels( topright, zoom)
def grid(self, z, x, y, fields, layer): """ Render the specified grid with Mapnik """ logger.debug(_("Render grid %s") % ((z, x, y),)) proj = GoogleProjection(self.tilesize, [z]) return self.render_grid(proj.tile_bbox((z, x, y)), fields, layer)
def tileslist(self, bbox, zoomlevels): """ Build the tiles list within the bottom-left/top-right bounding box (minx, miny, maxx, maxy) at the specified zoom levels. Return a list of tuples (z,x,y) """ proj = GoogleProjection(self.tile_size, zoomlevels, self.tile_scheme) return proj.tileslist(bbox)
def tileslist(self, bbox, zoomlevels, scheme='wmts'): """ Build the tiles list within the bottom-left/top-right bounding box (minx, miny, maxx, maxy) at the specified zoom levels. Return a list of tuples (z,x,y) """ proj = GoogleProjection(self.tile_size, zoomlevels, scheme) return proj.tileslist(bbox)
def tile(self, z, x, y): proj = GoogleProjection(self.tilesize, [z]) bbox = proj.tile_bbox((z, x, y)) bbox = proj.project(bbox[:2]) + proj.project(bbox[2:]) bbox = ','.join(map(str, bbox)) # Build WMS request URL encodedparams = urllib.urlencode(self.wmsParams) url = "%s?%s" % (self.url, encodedparams) url += "&bbox=%s" % bbox # commas are not encoded try: logger.debug(_("Download '%s'") % url) f = urllib.urlopen(url) header = f.info().typeheader assert header == self.wmsParams['format'], "Invalid WMS response type : %s" % header return f.read() except (AssertionError, IOError), e: raise ExtractionError
def tile(self, z, x, y): logger.debug(_("Request WMS tile %s") % ((z, x, y), )) proj = GoogleProjection(self.tilesize, [z]) bbox = proj.tile_bbox((z, x, y)) bbox = proj.project(bbox[:2]) + proj.project(bbox[2:]) bbox = ','.join(map(str, bbox)) # Build WMS request URL encodedparams = urllib.urlencode(self.wmsParams) url = "%s?%s" % (self.url, encodedparams) url += "&bbox=%s" % bbox # commas are not encoded try: logger.debug(_("Download '%s'") % url) f = urllib2.urlopen(url) header = f.info().typeheader assert header == self.wmsParams[ 'format'], "Invalid WMS response type : %s" % header return f.read() except (AssertionError, IOError): raise ExtractionError
def __init__(self, **kwargs): """ Manipulates tiles in general. Gives ability to list required tiles on a bounding box, download them, render them, extract them from other mbtiles... Keyword arguments: remote -- use remote tiles (default True) stylefile -- mapnik stylesheet file, only necessary if `remote` is `False` cache -- use a local cache to share tiles between runs (default True) tmp_dir -- temporary folder for gathering tiles (default DEFAULT_TMP_DIR) tiles_url -- remote URL to download tiles (default DEFAULT_TILES_URL) tile_size -- default tile size (default DEFAULT_TILE_SIZE) tiles_dir -- Local folder containing existing tiles, and where cached tiles will be stored (default DEFAULT_TILES_DIR) mbtiles_file -- A MBTiles providing tiles (overrides ``tiles_url``) """ self.remote = kwargs.get('remote', True) self.stylefile = kwargs.get('stylefile') self.tmp_dir = kwargs.get('tmp_dir', DEFAULT_TMP_DIR) self.cache = kwargs.get('cache', True) self.tiles_dir = kwargs.get('tiles_dir', DEFAULT_TILES_DIR) self.tiles_url = kwargs.get('tiles_url', DEFAULT_TILES_URL) self.tile_size = kwargs.get('tile_size', DEFAULT_TILE_SIZE) self.mbtiles_file = kwargs.get('mbtiles_file') if self.mbtiles_file: self.remote = False if not self.remote and not self.mbtiles_file: assert has_mapnik, "Cannot render tiles without mapnik !" assert self.stylefile, "A mapnik stylesheet is required" self.proj = GoogleProjection(self.tile_size) self._mapnik = None self._prj = None # Number of tiles rendered/downloaded here self.rendered = 0
def tileslist(self, bbox, zoomlevels): """ Build the tiles list within the bottom-left/top-right bounding box (minx, miny, maxx, maxy) at the specified zoom levels. Return a list of tuples (z,x,y) """ if len(bbox) != 4 or len(zoomlevels) == 0: raise InvalidCoverageError(_("Wrong format of bounding box or zoom levels.")) xmin, ymin, xmax, ymax = bbox if abs(xmin) > 180 or abs(xmax) > 180 or \ abs(ymin) > 90 or abs(ymax) > 90: raise InvalidCoverageError(_("Some coordinates exceed [-180,+180], [-90, 90].")) if xmin >= xmax or ymin >= ymax: raise InvalidCoverageError(_("Bounding box format is (xmin, ymin, xmax, ymax)")) if max(zoomlevels) >= self.proj.maxlevel: self.proj = GoogleProjection(self.tile_size, zoomlevels) ll0 = (xmin, ymax) # left top ll1 = (xmax, ymin) # right bottom l = [] for z in zoomlevels: px0 = self.proj.fromLLtoPixel(ll0,z) px1 = self.proj.fromLLtoPixel(ll1,z) for x in range(int(px0[0]/self.tile_size), int(px1[0]/self.tile_size)+1): if (x < 0) or (x >= 2**z): continue for y in range(int(px0[1]/self.tile_size), int(px1[1]/self.tile_size)+1): if (y < 0) or (y >= 2**z): continue l.append((z, x, y)) return l
class TilesManager(object): def __init__(self, **kwargs): """ Manipulates tiles in general. Gives ability to list required tiles on a bounding box, download them, render them, extract them from other mbtiles... Keyword arguments: remote -- use remote tiles (default True) stylefile -- mapnik stylesheet file, only necessary if `remote` is `False` cache -- use a local cache to share tiles between runs (default True) tmp_dir -- temporary folder for gathering tiles (default DEFAULT_TMP_DIR) tiles_url -- remote URL to download tiles (default DEFAULT_TILES_URL) tile_size -- default tile size (default DEFAULT_TILE_SIZE) tiles_dir -- Local folder containing existing tiles, and where cached tiles will be stored (default DEFAULT_TILES_DIR) mbtiles_file -- A MBTiles providing tiles (overrides ``tiles_url``) """ self.remote = kwargs.get('remote', True) self.stylefile = kwargs.get('stylefile') self.tmp_dir = kwargs.get('tmp_dir', DEFAULT_TMP_DIR) self.cache = kwargs.get('cache', True) self.tiles_dir = kwargs.get('tiles_dir', DEFAULT_TILES_DIR) self.tiles_url = kwargs.get('tiles_url', DEFAULT_TILES_URL) self.tile_size = kwargs.get('tile_size', DEFAULT_TILE_SIZE) self.mbtiles_file = kwargs.get('mbtiles_file') if self.mbtiles_file: self.remote = False if not self.remote and not self.mbtiles_file: assert has_mapnik, "Cannot render tiles without mapnik !" assert self.stylefile, "A mapnik stylesheet is required" self.proj = GoogleProjection(self.tile_size) self._mapnik = None self._prj = None # Number of tiles rendered/downloaded here self.rendered = 0 def tileslist(self, bbox, zoomlevels): """ Build the tiles list within the bottom-left/top-right bounding box (minx, miny, maxx, maxy) at the specified zoom levels. Return a list of tuples (z,x,y) """ if len(bbox) != 4 or len(zoomlevels) == 0: raise InvalidCoverageError() xmin, ymin, xmax, ymax = bbox if abs(xmin) > 180 or abs(xmax) > 180 or \ abs(ymin) > 90 or abs(ymax) > 90: raise InvalidCoverageError() if xmin >= xmax or ymin >= ymax: raise InvalidCoverageError() if max(zoomlevels) >= self.proj.maxlevel: self.proj = GoogleProjection(self.tile_size, zoomlevels) ll0 = (xmin, ymax) # left top ll1 = (xmax, ymin) # right bottom l = [] for z in zoomlevels: px0 = self.proj.fromLLtoPixel(ll0,z) px1 = self.proj.fromLLtoPixel(ll1,z) for x in range(int(px0[0]/self.tile_size), int(px1[0]/self.tile_size)+1): if (x < 0) or (x >= 2**z): continue for y in range(int(px0[1]/self.tile_size), int(px1[1]/self.tile_size)+1): if (y < 0) or (y >= 2**z): continue l.append((z, x, y)) return l def tile_file(self, (z, x, y)): """ Return folder (``z/x``) and name (``y.png``) for the specified tuple. """ tile_dir = os.path.join("%s" % z, "%s" % x) y_mercator = (2**z - 1) - y tile_name = "%s.png" % y_mercator return tile_dir, tile_name
def __init__(self, **kwargs): """ Manipulates tiles in general. Gives ability to list required tiles on a bounding box, download them, render them, extract them from other mbtiles... Keyword arguments: remote -- use remote tiles (default True) stylefile -- mapnik stylesheet file, only necessary if `remote` is `False` cache -- use a local cache to share tiles between runs (default True) tmp_dir -- temporary folder for gathering tiles (default DEFAULT_TMP_DIR) tiles_url -- remote URL to download tiles (default DEFAULT_TILES_URL) tile_size -- default tile size (default DEFAULT_TILE_SIZE) tiles_dir -- Local folder containing existing tiles, and where cached tiles will be stored (default DEFAULT_TILES_DIR) mbtiles_file -- A MBTiles providing tiles (overrides ``tiles_url``) wms_server -- A WMS server url wms_layers -- The list of layers to be requested wms_options -- WMS parameters to be requested (see ``landez.reader.WMSReader``) """ self.remote = kwargs.get('remote', True) self.stylefile = kwargs.get('stylefile') self.tmp_dir = kwargs.get('tmp_dir', DEFAULT_TMP_DIR) self.cache = kwargs.get('cache', True) self.tiles_dir = kwargs.get('tiles_dir', DEFAULT_TILES_DIR) self.tiles_url = kwargs.get('tiles_url', DEFAULT_TILES_URL) self.tiles_subdomains = kwargs.get('tiles_subdomains', DEFAULT_TILES_SUBDOMAINS) self.tile_size = kwargs.get('tile_size', DEFAULT_TILE_SIZE) self.mbtiles_file = kwargs.get('mbtiles_file') self.wms_server = kwargs.get('wms_server') self.wms_layers = kwargs.get('wms_layers', []) self.wms_options = kwargs.get('wms_options', {}) self.reader = None basename = '' if self.mbtiles_file: self.reader = MBTilesReader(self.mbtiles_file, self.tile_size) basename = os.path.basename(self.mbtiles_file) self.remote = False elif self.wms_server: assert self.wms_layers, _("Request at least one layer") self.reader = WMSReader(self.wms_server, self.wms_layers, self.tile_size, **self.wms_options) basename = '-'.join(self.wms_layers) self.remote = False elif self.stylefile: assert has_mapnik, _("Cannot render tiles without mapnik !") assert self.stylefile, _("A mapnik stylesheet is required") self.remote = False basename = os.path.basename(self.stylefile) else: url = urlparse(self.tiles_url) basename = url.netloc basename = re.sub(r'[^a-z^A-Z^0-9]+', '', basename) self.tmp_dir = os.path.join(self.tmp_dir, basename) self.tiles_dir = os.path.join(self.tiles_dir, basename) self.proj = GoogleProjection(self.tile_size) self._mapnik = None self._prj = None self._reader = None self._layers = [] # Number of tiles rendered/downloaded here self.rendered = 0
class TilesManager(object): def __init__(self, **kwargs): """ Manipulates tiles in general. Gives ability to list required tiles on a bounding box, download them, render them, extract them from other mbtiles... Keyword arguments: remote -- use remote tiles (default True) stylefile -- mapnik stylesheet file, only necessary if `remote` is `False` cache -- use a local cache to share tiles between runs (default True) tmp_dir -- temporary folder for gathering tiles (default DEFAULT_TMP_DIR) tiles_url -- remote URL to download tiles (default DEFAULT_TILES_URL) tile_size -- default tile size (default DEFAULT_TILE_SIZE) tiles_dir -- Local folder containing existing tiles, and where cached tiles will be stored (default DEFAULT_TILES_DIR) mbtiles_file -- A MBTiles providing tiles (overrides ``tiles_url``) wms_server -- A WMS server url wms_layers -- The list of layers to be requested wms_options -- WMS parameters to be requested (see ``landez.reader.WMSReader``) """ self.remote = kwargs.get('remote', True) self.stylefile = kwargs.get('stylefile') self.tmp_dir = kwargs.get('tmp_dir', DEFAULT_TMP_DIR) self.cache = kwargs.get('cache', True) self.tiles_dir = kwargs.get('tiles_dir', DEFAULT_TILES_DIR) self.tiles_url = kwargs.get('tiles_url', DEFAULT_TILES_URL) self.tiles_subdomains = kwargs.get('tiles_subdomains', DEFAULT_TILES_SUBDOMAINS) self.tile_size = kwargs.get('tile_size', DEFAULT_TILE_SIZE) self.mbtiles_file = kwargs.get('mbtiles_file') self.wms_server = kwargs.get('wms_server') self.wms_layers = kwargs.get('wms_layers', []) self.wms_options = kwargs.get('wms_options', {}) self.reader = None basename = '' if self.mbtiles_file: self.reader = MBTilesReader(self.mbtiles_file, self.tile_size) basename = os.path.basename(self.mbtiles_file) self.remote = False elif self.wms_server: assert self.wms_layers, _("Request at least one layer") self.reader = WMSReader(self.wms_server, self.wms_layers, self.tile_size, **self.wms_options) basename = '-'.join(self.wms_layers) self.remote = False elif self.stylefile: assert has_mapnik, _("Cannot render tiles without mapnik !") assert self.stylefile, _("A mapnik stylesheet is required") self.remote = False basename = os.path.basename(self.stylefile) else: url = urlparse(self.tiles_url) basename = url.netloc basename = re.sub(r'[^a-z^A-Z^0-9]+', '', basename) self.tmp_dir = os.path.join(self.tmp_dir, basename) self.tiles_dir = os.path.join(self.tiles_dir, basename) self.proj = GoogleProjection(self.tile_size) self._mapnik = None self._prj = None self._reader = None self._layers = [] # Number of tiles rendered/downloaded here self.rendered = 0 def tileslist(self, bbox, zoomlevels): """ Build the tiles list within the bottom-left/top-right bounding box (minx, miny, maxx, maxy) at the specified zoom levels. Return a list of tuples (z,x,y) """ if len(bbox) != 4 or len(zoomlevels) == 0: raise InvalidCoverageError(_("Wrong format of bounding box or zoom levels.")) xmin, ymin, xmax, ymax = bbox if abs(xmin) > 180 or abs(xmax) > 180 or \ abs(ymin) > 90 or abs(ymax) > 90: raise InvalidCoverageError(_("Some coordinates exceed [-180,+180], [-90, 90].")) if xmin >= xmax or ymin >= ymax: raise InvalidCoverageError(_("Bounding box format is (xmin, ymin, xmax, ymax)")) if max(zoomlevels) >= self.proj.maxlevel: self.proj = GoogleProjection(self.tile_size, zoomlevels) ll0 = (xmin, ymax) # left top ll1 = (xmax, ymin) # right bottom l = [] for z in zoomlevels: px0 = self.proj.fromLLtoPixel(ll0,z) px1 = self.proj.fromLLtoPixel(ll1,z) for x in range(int(px0[0]/self.tile_size), int(px1[0]/self.tile_size)+1): if (x < 0) or (x >= 2**z): continue for y in range(int(px0[1]/self.tile_size), int(px1[1]/self.tile_size)+1): if (y < 0) or (y >= 2**z): continue l.append((z, x, y)) return l def tile_file(self, (z, x, y)): """ Return folder (``z/x``) and name (``y.png``) for the specified tuple. """ tile_dir = os.path.join("%s" % z, "%s" % x) y_mercator = (2**z - 1) - y tile_name = "%s.png" % y_mercator return tile_dir, tile_name
def disk_to_mbtiles(directory_path, mbtiles_file, **kwargs): logger.info("Importing disk to MBTiles") logger.debug("%s --> %s" % (directory_path, mbtiles_file)) con = mbtiles_connect(mbtiles_file) cur = con.cursor() optimize_connection(cur) mbtiles_setup(cur) #~ image_format = 'png' image_format = kwargs.get('format', 'png') try: metadata = json.load(open(os.path.join(directory_path, 'metadata.json'), 'r')) image_format = kwargs.get('format') for name, value in metadata.items(): cur.execute('insert into metadata (name, value) values (?, ?)', (name, value)) logger.info('metadata from metadata.json restored') except IOError: logger.warning('metadata.json not found') count = 0 start_time = time.time() msg = "" tile_range = None if 'bbox' in kwargs and kwargs['bbox'] is not None: bounds_string = ",".join([str(f) for f in kwargs['bbox']]) cur.execute('delete from metadata where name = ?', ('bounds',)) cur.execute('insert into metadata (name, value) values (?, ?)', ('bounds', bounds_string)) logger.info("Using bbox " + bounds_string) zoom_range = kwargs.get("zoom_range", range(0, 22)) proj = GoogleProjection(256, zoom_range, "tms") tile_range = proj.tileranges(kwargs['bbox']) for z in sorted(tile_range.keys()): logger.info("z:%i x:%i-%i y:%i-%i" % (z, tile_range[z]['x'][0], tile_range[z]['x'][1], tile_range[z]['y'][0], tile_range[z]['y'][1])) for zoomDir in getDirs(directory_path): if kwargs.get("scheme") == 'ags': if not "L" in zoomDir: logger.warning("You appear to be using an ags scheme on an non-arcgis Server cache.") z = int(zoomDir.replace("L", "")) else: if "L" in zoomDir: logger.warning("You appear to be using a %s scheme on an arcgis Server cache. Try using --scheme=ags instead" % kwargs.get("scheme")) try: z = int(zoomDir) except: logger.info("Skipping dir " + zoomDir) continue if tile_range and not z in tile_range: logger.debug('Skipping zoom level %i' % (z,)) continue for rowDir in getDirs(os.path.join(directory_path, zoomDir)): if kwargs.get("scheme") == 'ags': y = flip_y(z, int(rowDir.replace("R", ""), 16)) elif kwargs.get("scheme") == 'zyx': y = flip_y(z, int(rowDir)) else: x = int(rowDir) for current_file in os.listdir(os.path.join(directory_path, zoomDir, rowDir)): file_name, ext = current_file.split('.',1) f = open(os.path.join(directory_path, zoomDir, rowDir, current_file), 'rb') file_content = f.read() f.close() if kwargs.get('scheme') == 'xyz': y = flip_y(int(z), int(file_name)) elif kwargs.get("scheme") == 'ags': x = int(file_name.replace("C", ""), 16) elif kwargs.get("scheme") == 'zyx': x = int(file_name) else: y = int(file_name) if tile_range: r = tile_range[z] if x < r['x'][0] or x > r['x'][1] or y < r['y'][0] or y > r['y'][1]: logger.debug(' Skipping tile Zoom (z): %i\tCol (x): %i\tRow (y): %i' % (z, x, y)) continue if (ext == image_format): logger.debug(' Read tile from Zoom (z): %i\tCol (x): %i\tRow (y): %i' % (z, x, y)) cur.execute("""insert into tiles (zoom_level, tile_column, tile_row, tile_data) values (?, ?, ?, ?);""", (z, x, y, sqlite3.Binary(file_content))) count = count + 1 if (count % 100) == 0: for c in msg: sys.stdout.write(chr(8)) msg = "%s tiles inserted (%d tiles/sec)" % (count, count / (time.time() - start_time)) sys.stdout.write(msg) elif (ext == 'grid.json'): logger.debug(' Read grid from Zoom (z): %i\tCol (x): %i\tRow (y): %i' % (z, x, y)) # Remove potential callback with regex file_content = file_content.decode('utf-8') has_callback = re.match(r'[\w\s=+-/]+\(({(.|\n)*})\);?', file_content) if has_callback: file_content = has_callback.group(1) utfgrid = json.loads(file_content) data = utfgrid.pop('data') compressed = zlib.compress(json.dumps(utfgrid).encode()) cur.execute("""insert into grids (zoom_level, tile_column, tile_row, grid) values (?, ?, ?, ?) """, (z, x, y, sqlite3.Binary(compressed))) grid_keys = [k for k in utfgrid['keys'] if k != ""] for key_name in grid_keys: key_json = data[key_name] cur.execute("""insert into grid_data (zoom_level, tile_column, tile_row, key_name, key_json) values (?, ?, ?, ?, ?);""", (z, x, y, key_name, json.dumps(key_json))) logger.debug('tiles (and grids) inserted.') optimize_database(con)