def _fuzzyTileFileExists(self, lzxy):
"""Try to find a tile image file for the given coordinates."""
# first check if the primary tile path exists
tilePath = self._getTileFilePath(lzxy)
# check if the primary file path exists and also if it actually
# is an image file
if os.path.exists(tilePath):
try:
with open(tilePath, "rb") as f:
if utils.isTheStringAnImage(f.read(32)):
return tilePath
else:
self.log.warning("%s is not an image", tilePath)
except Exception:
self.log("error when checking if primary tile file is an image")
# look also for other supported image formats
alternativeTilePath = None
# TODO: might be good to investigate the performance impact,
# just to be sure :P
# replace the extension with a * so that the path can be used
# as wildcard in glob
wildcardTilePath = "%s.*" % os.path.splitext(tilePath)[0]
# iterate over potential paths returned by the glob iterator
for path in glob.iglob(wildcardTilePath):
# go over the paths and check if they point to an image files
with open(path, "rb") as f:
if utils.isTheStringAnImage(f.read(32)):
# once an image file is found, break & returns its path
alternativeTilePath = path
break
else:
self.log.warning("%s is not an image", tilePath)
return alternativeTilePath
def downloadTile(self, lzxy):
"""Download the a tile from the network
:param tuple lzxy: tile description tuple
:returns: tile data or None
:rtype: data or None
"""
tileUrl = tiles.getTileUrl(lzxy)
# print("GET TILE")
# print(tileUrl)
response = self._getConnPool(lzxy[0].id, tileUrl).request('GET', tileUrl)
# print("RESPONSE")
# print(response)
tileData = response.data
if tileData:
# check if the data is actually an image, and not an error page
if utils.isTheStringAnImage(tileData):
self._storeTiles.automaticStoreTile(tileData, lzxy)
# print("STORED")
return tileData
else:
print("mapTiles: tile data returned by remote tileserver was not an image")
print("layer:%s z:%d x:%d y:%d" % (lzxy[0].id, lzxy[1], lzxy[2], lzxy[3]))
print("tile url: %s" % tileUrl)
print("NOTE: this probably means that the tileserver returned an"
"error page in place of the tile, because it doesn't like you")
return None
else:
return None
def getTileFromDb(self, lookupConn, dbFolderPath, lzxy):
"""get a tile from the database"""
accessType = "get"
layer, z, x, y = lzxy
#look in the lookup db
#with self.lookupConnectionLock:
if 1:
# just to make sure the access is sequential
# (due to sqlite in python 2.5 probably not liking concurrent access,
# resulting in te database becoming unavailable)
lookupCursor = lookupConn.cursor()
lookupResult = lookupCursor.execute(
"select store_filename, unix_epoch_timestamp from tiles where z=? and x=? and y=?",
(z, x, y)).fetchone()
if lookupResult: # the tile was found in the lookup db
# now search in the store
storeFilename = lookupResult[0]
pathToStore = self.getStorePath(dbFolderPath, storeFilename)
stores = self.layers[accessType][dbFolderPath]['stores']
connectionToStore = self.connectToStore(
stores, pathToStore, dbFolderPath, accessType)
storeCursor = connectionToStore.cursor()
# as the x,y & z are used as the primary key, all rows need to have a unique
# x, y & z combination and thus there can be only one result for a select
# over x, y & z
result = storeCursor.execute(
"select tile, unix_epoch_timestamp from tiles where z=? and x=? and y=?",
(z, x, y))
if utils.isTheStringAnImage(str(result)):
self.log.warning("%s,%s,%s in %s is probably not an image",
x, y, z, dbFolderPath)
return result
else: # the tile was not found in the lookup table
return None
def getTileFromDb(self, lookupConn, dbFolderPath, lzxy):
"""get a tile from the database"""
accessType = "get"
layer, z, x, y = lzxy
#look in the lookup db
#with self.lookupConnectionLock:
if 1:
# just to make sure the access is sequential
# (due to sqlite in python 2.5 probably not liking concurrent access,
# resulting in te database becoming unavailable)
lookupCursor = lookupConn.cursor()
lookupResult = lookupCursor.execute(
"select store_filename, unix_epoch_timestamp from tiles where z=? and x=? and y=?",
(z, x, y)).fetchone()
if lookupResult: # the tile was found in the lookup db
# now search in the store
storeFilename = lookupResult[0]
pathToStore = self.getStorePath(dbFolderPath, storeFilename)
stores = self.layers[accessType][dbFolderPath]['stores']
connectionToStore = self.connectToStore(stores, pathToStore, dbFolderPath, accessType)
storeCursor = connectionToStore.cursor()
# as the x,y & z are used as the primary key, all rows need to have a unique
# x, y & z combination and thus there can be only one result for a select
# over x, y & z
result = storeCursor.execute(
"select tile, unix_epoch_timestamp from tiles where z=? and x=? and y=?",
(z, x, y))
if utils.isTheStringAnImage(str(result)):
self.log.warning("%s,%s,%s in %s is probably not an image", x, y, z, dbFolderPath)
return result
else: # the tile was not found in the lookup table
return None
def _fuzzyTileFileExists(self, lzxy):
"""Try to find a tile image file for the given coordinates."""
# first check if the primary tile path exists
tilePath = self._getTileFilePath(lzxy)
# check if the primary file path exists and also if it actually
# is an image file
if os.path.exists(tilePath):
try:
with open(tilePath, "rb") as f:
if utils.isTheStringAnImage(f.read(32)):
return tilePath
else:
self.log.warning("%s is not an image", tilePath)
except Exception:
self.log(
"error when checking if primary tile file is an image")
# look also for other supported image formats
alternativeTilePath = None
# TODO: might be good to investigate the performance impact,
# just to be sure :P
# replace the extension with a * so that the path can be used
# as wildcard in glob
wildcardTilePath = "%s.*" % os.path.splitext(tilePath)[0]
# iterate over potential paths returned by the glob iterator
for path in glob.iglob(wildcardTilePath):
# go over the paths and check if they point to an image files
with open(path, "rb") as f:
if utils.isTheStringAnImage(f.read(32)):
# once an image file is found, break & returns its path
alternativeTilePath = path
break
else:
self.log.warning("%s is not an image", tilePath)
return alternativeTilePath
def saveTileForURL(self, tile):
"""save a tile for url created from its coordinates"""
lzxy = (self.layer, tile[2], tile[0], tile[1])
(url, filename, folder) = self.callback.getTileUrlAndPath(lzxy)
m = self.callback.m.get('storeTiles', None) # get the tile storage module
if m:
goAhead = False
redownload = int(self.callback.get('batchRedownloadAvailableTiles', False))
if not redownload:
# does the the file exist ?
# -> don't download it if it does
goAhead = not m.tileExists2(lzxy, fromThread=True)
elif redownload == 1: # redownload all
goAhead = True
elif redownload == 2: # update
# only download tiles in the area that already exist
goAhead = m.tileExists2(filename, lzxy, fromThread=True)
# TODO: maybe make something like tile objects so we don't have to pass so many parameters ?
if goAhead: # if the file does not exist
request = self.connPool.request('get', url)
size = int(request.getheaders()['content-length'])
content = request.data
# The tileserver sometimes returns a HTML error page
# instead of the tile, which is then saved instead of the tile an
# users are then confused why tiles they have downloaded don't show up.
# To raise a proper error on this behaviour, we check the tiles magic number
# and if is not an image we raise the TileNotImageException.
# TODO: does someone supply non-bitmap/SVG tiles ?
if utils.isTheStringAnImage(content):
#its an image, save it
m.automaticStoreTile(content, lzxy)
else:
# its not ana image, raise exception
raise TileNotImageException()
return size # something was actually downloaded and saved
else:
return False # nothing was downloaded
def _saveTileForURL(self, lzxy):
"""save a tile for url created from its coordinates"""
url = tiles.getTileUrl(lzxy)
goAhead = False
redownload = int(modrana.get('batchRedownloadAvailableTiles', False))
# TODO: use constants for the ENUM
if not redownload:
# does the the file exist ?
# -> don't download it if it does
goAhead = not self._storeTiles.tileExists2(lzxy, fromThread=True)
elif redownload == 1: # redownload all
goAhead = True
elif redownload == 2: # update
# only download tiles in the area that already exist
goAhead = self._storeTiles.tileExists2(lzxy, fromThread=True)
if goAhead: # if the file does not exist
request = self._connPool.request('get', url)
size = int(request.getheaders()['content-length'])
content = request.data
# The tileserver sometimes returns a HTML error page
# instead of the tile, which is then saved instead of the tile an
# users are then confused why tiles they have downloaded don't show up.
# To raise a proper error on this behaviour, we check the tiles magic number
# and if is not an image we raise the TileNotImageException.
# TODO: does someone supply non-bitmap/SVG tiles ?
if utils.isTheStringAnImage(content):
#its an image, save it
self._storeTiles.automaticStoreTile(content, lzxy)
else:
# its not ana image, raise exception
raise TileNotImageException(url)
return size # something was actually downloaded and saved
else:
return False # nothing was downloaded