def _downloadTile(self, lzxy):
"""Download the a tile from the network
:param tuple lzxy: tile description tuple
:returns: tile data or None
:rtype: data or None
"""
tileUrl = tiles.getTileUrl(lzxy)
# self.log.debug("GET TILE")
# self.log.debug(tileUrl)
response = self._getConnPool(lzxy[0], tileUrl).request('GET', tileUrl)
# self.log.debug("RESPONSE")
# self.log.debug(response)
tileData = response.data
if tileData:
# check if the data is actually an image, and not an error page
if utils.is_the_string_an_image(tileData):
self._storeTiles.store_tile_data(lzxy, tileData)
# self.log.debug("STORED")
return tileData
else:
msg = "tile data returned by remote tileserver was not an image\n"
msg+= "layer:%s z:%d x:%d y:%d\n" % (lzxy[0].id, lzxy[1], lzxy[2], lzxy[3])
msg+= "tile url: %s\n" % tileUrl
msg+= "NOTE: this probably means that the tileserver returned an\n"
msg+= "error page in place of the tile, because it doesn't like you\n"
self.log.warning(msg)
return None
else:
return None
def _checkTileSize(self, lzxy):
"""Get a size of a tile from HTTP header,
if the tile is locally available remove it form the
download request set
:returns: size in bytes, None if tile is available and
0 if the header check raised an exception
:rtype: int or None
"""
size = 0
url = "unknown url"
try:
url = tiles.getTileUrl(lzxy)
# does the tile exist ?
if self._storeTiles.tileExists2(lzxy, fromThread=True): # if the file does not exist
size = None # it exists, return None
else:
# the tile does not exist, get its HTTP header
request = self._connPool.urlopen('HEAD', url)
size = int(request.getheaders()['content-length'])
except IOError:
log.error("Could not open document: %s", url)
# the url errored out, so we just say it has zero size
size = 0
except Exception:
log.exception("error, while checking size of tile: %s", lzxy)
size = 0
return size
def downloadTile(self, lzxy):
"""Download the a tile from the network
:param tuple lzxy: tile description tuple
:returns: tile data or None
:rtype: data or None
"""
tileUrl = tiles.getTileUrl(lzxy)
# print("GET TILE")
# print(tileUrl)
response = self._getConnPool(lzxy[0].id, tileUrl).request('GET', tileUrl)
# print("RESPONSE")
# print(response)
tileData = response.data
if tileData:
# check if the data is actually an image, and not an error page
if utils.isTheStringAnImage(tileData):
self._storeTiles.automaticStoreTile(tileData, lzxy)
# print("STORED")
return tileData
else:
print("mapTiles: tile data returned by remote tileserver was not an image")
print("layer:%s z:%d x:%d y:%d" % (lzxy[0].id, lzxy[1], lzxy[2], lzxy[3]))
print("tile url: %s" % tileUrl)
print("NOTE: this probably means that the tileserver returned an"
"error page in place of the tile, because it doesn't like you")
return None
else:
return None
def getTileUrlAndPath(self, lzxy):
mapTiles = self.m.get('mapTiles', None)
url = tiles.getTileUrl(lzxy) # generate url
tileFolder = self._getTileFolderPath() # where should we store the downloaded tiles
filePath = os.path.join(tileFolder, mapTiles.getImagePath(lzxy))
fileFolder = os.path.join(tileFolder, mapTiles.getImageFolder(lzxy))
return url, filePath, fileFolder
def _printErrorMessage(self, e, lzxy):
url = tiles.getTileUrl(lzxy)
error = "mapTiles: download thread reports error\n"
error += "** we were doing this, when an exception occurred:\n"
error += "** downloading tile: x:%d,y:%d,z:%d, layer:%s, url: %s" % (
lzxy[1], lzxy[2], lzxy[3], lzxy[0].id, url)
log.exception(error)
def _printErrorMessage(self, e, lzxy):
url = tiles.getTileUrl(lzxy)
error = "mapTiles: download thread reports error\n"
error+= "** we were doing this, when an exception occurred:\n"
error+= "** downloading tile: x:%d,y:%d,z:%d, layer:%s, url: %s" % (
lzxy[1],
lzxy[2],
lzxy[3],
lzxy[0].id,
url)
log.exception(error)
def getAnUrl(batch, layer):
"""Get a random url from the set so we can init the pool"""
if batch:
tile = None
for t in batch:
tile = t
break
(x, y, z) = (tile[0], tile[1], tile[2])
url = tiles.getTileUrl((layer, z, x, y))
else:
url = ""
return url
def getAnUrl(self, neededTiles):
"""get a random url so we can init the pool"""
if neededTiles:
tile = None
for t in neededTiles:
tile = t
break
(x, y, z) = (tile[0], tile[1], tile[2])
url = tiles.getTileUrl((self.layer, z, x, y))
else:
url = ""
return url
def _printErrorMessage(self, e, lzxy):
import traceback
url = tiles.getTileUrl(lzxy)
print("mapTiles: download thread reports error")
print("** we were doing this, when an exception occurred:")
print("** downloading tile: x:%d,y:%d,z:%d, layer:%s, url: %s" % (
lzxy[1],
lzxy[2],
lzxy[3],
lzxy[0].id,
url))
print("** this exception occurred: %s\n" % e)
print("** traceback:\n")
traceback.print_exc()
def _saveTileForURL(self, lzxy):
"""save a tile for url created from its coordinates"""
url = tiles.getTileUrl(lzxy)
goAhead = False
redownload = int(modrana.get('batchRedownloadAvailableTiles', False))
# TODO: use constants for the ENUM
if not redownload:
# does the the file exist ?
# -> don't download it if it does
goAhead = not self._storeTiles.tileExists2(lzxy, fromThread=True)
elif redownload == 1: # redownload all
goAhead = True
elif redownload == 2: # update
# only download tiles in the area that already exist
goAhead = self._storeTiles.tileExists2(lzxy, fromThread=True)
if goAhead: # if the file does not exist
request = self._connPool.request('get', url)
size = int(request.getheaders()['content-length'])
content = request.data
# The tileserver sometimes returns a HTML error page
# instead of the tile, which is then saved instead of the tile an
# users are then confused why tiles they have downloaded don't show up.
# To raise a proper error on this behaviour, we check the tiles magic number
# and if is not an image we raise the TileNotImageException.
# TODO: does someone supply non-bitmap/SVG tiles ?
if utils.isTheStringAnImage(content):
#its an image, save it
self._storeTiles.automaticStoreTile(content, lzxy)
else:
# its not ana image, raise exception
raise TileNotImageException(url)
return size # something was actually downloaded and saved
else:
return False # nothing was downloaded
def getTileUrl(self, x, y, z, layerId):
"""Return url for given tile coordinates and layer"""
layer = self._mapLayersModule.getLayerById(layerId)
# TODO: make this to work with layer objects directly
return tiles.getTileUrl((layer, z, x, y))