def __init__(self, maxThreads, taskBufferSize=0,
taskTimeout=0):
self._mapTiles = modrana.m.get("mapTiles")
self._storeTiles = modrana.m.get("storeTiles")
# if task buffer size is set, start leaking
# old tile download requests from the bottom of the
# request stack once it becomes full, as we don't want
# the work queue to block and discarding old tile
# download requests is not an issue
leak = taskBufferSize >= 0
self._pool = LifoThreadPool(maxThreads,
name=constants.THREAD_POOL_AUTOMATIC_TILE_DOWNLOAD,
taskBufferSize=taskBufferSize,
leak=leak)
# in seconds, 0 == no task timeout
self._taskTimeout = taskTimeout
self._running = set()
self._runningLock = threading.RLock()
# due to GIL, we don't have a lock
# for the set of tile downloads
# in progress (hopefully)
self._imageSurface = self._mapTiles.cacheImageSurfaces
def __init__(self, maxThreads, taskBufferSize=0,
taskTimeout=0):
self._mapTiles = modrana.m.get("mapTiles")
self._storeTiles = modrana.m.get("storeTiles")
# if task buffer size is set, start leaking
# old tile download requests from the bottom of the
# request stack once it becomes full, as we don't want
# the work queue to block and discarding old tile
# download requests is not an issue
leak = taskBufferSize >= 0
self._pool = LifoThreadPool(maxThreads,
name=constants.THREAD_POOL_AUTOMATIC_TILE_DOWNLOAD,
taskBufferSize=taskBufferSize,
leak=leak)
# in seconds, 0 == no task timeout
self._taskTimeout = taskTimeout
self._running = set()
self._runningLock = threading.RLock()
# due to GIL, we don't have a lock
# for the set of tile downloads
# in progress (hopefully)
self._imageSurface = self._mapTiles.cacheImageSurfaces
class Downloader(object):
def __init__(self, maxThreads, taskBufferSize=0,
taskTimeout=0):
self._mapTiles = modrana.m.get("mapTiles")
self._storeTiles = modrana.m.get("storeTiles")
# if task buffer size is set, start leaking
# old tile download requests from the bottom of the
# request stack once it becomes full, as we don't want
# the work queue to block and discarding old tile
# download requests is not an issue
leak = taskBufferSize >= 0
self._pool = LifoThreadPool(maxThreads,
name=constants.THREAD_POOL_AUTOMATIC_TILE_DOWNLOAD,
taskBufferSize=taskBufferSize,
leak=leak)
# in seconds, 0 == no task timeout
self._taskTimeout = taskTimeout
self._running = set()
self._runningLock = threading.RLock()
# due to GIL, we don't have a lock
# for the set of tile downloads
# in progress (hopefully)
self._imageSurface = self._mapTiles.cacheImageSurfaces
def shutdown(self):
self._pool.shutdown(now=True)
def _tileDownloaded(self, error, lzxy, tag):
#log.debug("DOWNLOADER: CALLING SIGNAL: %s %s" % (tag, success))
self._mapTiles.tileDownloaded(error, lzxy, tag)
def downloadTile(self, lzxy, tag=None, overwrite=False):
"""Add a tile download request, if this download
request replaces another not yet handled request
from the bottom of the work stack, the old
lzxy will be returned
:param tuple lzxy: tile to download represented by a tuple
:param str tag: tracking tag for the download request
:param bool overwrite: download tile even if locally available
:returns: None or a tuple that was removed because of the new one
:rtype: None or tuple
"""
discardedRequest = None
discardedTile = None
with self._runningLock:
if lzxy not in self._running:
# drop download requests for tiles that are already
# being downloaded
discardedRequest = self._pool.submit(
self._handleDownload, lzxy, tag, time.time(), overwrite
)
# return lzxy & tag for any discarded request or return None
# if no request was discarded
if discardedRequest:
discardedTile = discardedRequest[1][0], discardedRequest[1][1]
return discardedTile
def _handleDownload(self, lzxy, tag, timestamp, overwrite):
download = True
error = constants.TILE_DOWNLOAD_ERROR
with self._runningLock:
if (lzxy, tag) in self._running:
# tile is already being downloaded
download = False
else:
# tile is not yet being downloaded
# so register we are handling it
self._running.add((lzxy, tag))
if self._taskTimeout:
dt = time.time() - timestamp
if dt >= self._taskTimeout:
# download request timed out
download = False
if not download and not overwrite:
# check if the tile has been already downloaded
download = not self._storeTiles.tileExists2(lzxy, fromThread=True)
if download:
# download tile
try:
self._downloadTile(lzxy)
error = constants.TILE_DOWNLOAD_SUCCESS
except urllib3.exceptions.HTTPError:
# server returned a HTTP error, this means we got
# to the server but it didn't like us for some reason,
error = self._fatalDownloadError(lzxy)
except URLError:
# this is most probably caused by a loss of network connectivity
error = self._temporaryDownloadError(lzxy)
# something other is wrong (most probably a corrupt tile)
except Exception:
import sys
e = sys.exc_info()[1]
self._printErrorMessage(e, lzxy)
# remove the status tile
self._mapTiles.removeImageFromMemory(lzxy)
error = constants.TILE_DOWNLOAD_ERROR
finally:
# done, unregister the tile from the tracking set
with self._runningLock:
try:
self._running.remove((lzxy, tag))
except KeyError:
pass
# TODO: find why this happens (well, it appears to be harmless)
# and maybe forward it to debug log once we have one ?
#print("auto tile dl pool: warning, tuple already removed from tracking!")
#print(lzxy)
# report that tha tile has or has not bee successfully downloaded
self._tileDownloaded(error, lzxy, tag)
else:
# don't download tile and remove
# any "downloading" tiles that might
# be in the image cache
self._mapTiles.removeImageFromMemory(lzxy)
# report the tile as not been downloaded
self._tileDownloaded(error, lzxy, tag)
def _downloadTile(self, lzxy):
"""Downloads a tile image image from network"""
self._downloadInProgress(lzxy)
content = self._mapTiles._downloadTile(lzxy)
if content is None:
raise urllib3.exceptions.HTTPError
if self._imageSurface:
pl = gtk.gdk.PixbufLoader()
pl.write(content)
pl.close() # this blocks until the image is completely loaded
# http://www.ossramblings.com/loading_jpg_into_cairo_surface_python
surface = self._mapTiles._pixbuf2cairoImageSurface(pl.get_pixbuf())
self._mapTiles.storeInMemory(surface, lzxy)
# like this, corrupted tiles should not get past the pixbuf loader and be stored
else:
# cache the raw data
self._mapTiles.storeInMemory(content, lzxy)
self._storeTiles.automaticStoreTile(content, lzxy)
def _downloadInProgress(self, lzxy):
if self._imageSurface:
# change the status tile to "Downloading..."
self._mapTiles.storeInMemory(self._mapTiles.downloadingTile[0], lzxy, imageType="downloading")
def _temporaryDownloadError(self, lzxy):
if self._imageSurface:
tileNetworkErrorSurface = self._mapTiles.images[1]['tileNetworkError'][0]
expireTimestamp = time.time() + 10
self._mapTiles.storeInMemory(tileNetworkErrorSurface, lzxy, 'error',
expireTimestamp) # retry after 10 seconds
# as not to DOS the system when we temporarily loose internet connection or other such error
# occurs, we load a temporary error tile with expiration timestamp instead of the tile image
# TODO: actually remove tiles according to expiration timestamp :)
return constants.TILE_DOWNLOAD_TEMPORARY_ERROR
def _fatalDownloadError(self, lzxy):
if self._imageSurface:
tileDownloadFailedSurface = self._mapTiles.images[1]['tileDownloadFailed'][0]
expireTimestamp = time.time() + 10
self._mapTiles.storeInMemory(tileDownloadFailedSurface, lzxy, 'semiPermanentError',
expireTimestamp)
# like this, when tile download fails due to a http error,
# the error tile is loaded instead
# like this:
# - modRana does not immediately try to download a tile that errors out
# - the error tile is shown without modifying the pipeline too much
# - modRana will eventually try to download the tile again,
# after it is flushed with old tiles from the memory
return constants.TILE_DOWNLOAD_ERROR
def _printErrorMessage(self, e, lzxy):
url = tiles.getTileUrl(lzxy)
error = "mapTiles: download thread reports error\n"
error+= "** we were doing this, when an exception occurred:\n"
error+= "** downloading tile: x:%d,y:%d,z:%d, layer:%s, url: %s" % (
lzxy[1],
lzxy[2],
lzxy[3],
lzxy[0].id,
url)
log.exception(error)
@property
def maxThreads(self):
return self._pool.maxThreads
@property
def qsize(self):
return self._pool.qsize()
class Downloader(object):
def __init__(self, maxThreads, taskBufferSize=0,
taskTimeout=0):
self._mapTiles = modrana.m.get("mapTiles")
self._storeTiles = modrana.m.get("storeTiles")
# if task buffer size is set, start leaking
# old tile download requests from the bottom of the
# request stack once it becomes full, as we don't want
# the work queue to block and discarding old tile
# download requests is not an issue
leak = taskBufferSize >= 0
self._pool = LifoThreadPool(maxThreads,
name=constants.THREAD_POOL_AUTOMATIC_TILE_DOWNLOAD,
taskBufferSize=taskBufferSize,
leak=leak)
# in seconds, 0 == no task timeout
self._taskTimeout = taskTimeout
self._running = set()
self._runningLock = threading.RLock()
# due to GIL, we don't have a lock
# for the set of tile downloads
# in progress (hopefully)
self._imageSurface = self._mapTiles.cacheImageSurfaces
def shutdown(self):
self._pool.shutdown(now=True)
def _tileDownloaded(self, error, lzxy, tag):
#log.debug("DOWNLOADER: CALLING SIGNAL: %s %s" % (tag, success))
self._mapTiles.tileDownloaded(error, lzxy, tag)
def downloadTile(self, lzxy, tag=None, overwrite=False):
"""Add a tile download request, if this download
request replaces another not yet handled request
from the bottom of the work stack, the old
lzxy will be returned
:param tuple lzxy: tile to download represented by a tuple
:param str tag: tracking tag for the download request
:param bool overwrite: download tile even if locally available
:returns: None or a tuple that was removed because of the new one
:rtype: None or tuple
"""
discardedRequest = None
discardedTile = None
with self._runningLock:
if lzxy not in self._running:
# drop download requests for tiles that are already
# being downloaded
discardedRequest = self._pool.submit(
self._handleDownload, lzxy, tag, time.time(), overwrite
)
# return lzxy & tag for any discarded request or return None
# if no request was discarded
if discardedRequest:
discardedTile = discardedRequest[1][0], discardedRequest[1][1]
return discardedTile
def _handleDownload(self, lzxy, tag, timestamp, overwrite):
download = True
error = constants.TILE_DOWNLOAD_ERROR
with self._runningLock:
if (lzxy, tag) in self._running:
# tile is already being downloaded
download = False
else:
# tile is not yet being downloaded
# so register we are handling it
self._running.add((lzxy, tag))
if self._taskTimeout:
dt = time.time() - timestamp
if dt >= self._taskTimeout:
# download request timed out
download = False
if not download and not overwrite:
# check if the tile has been already downloaded
download = not self._storeTiles.tile_is_stored(lzxy)
if download:
# download tile
try:
self._downloadTile(lzxy)
error = constants.TILE_DOWNLOAD_SUCCESS
except urllib3.exceptions.HTTPError:
# server returned a HTTP error, this means we got
# to the server but it didn't like us for some reason,
error = self._fatalDownloadError(lzxy)
except URLError:
# this is most probably caused by a loss of network connectivity
error = self._temporaryDownloadError(lzxy)
# something other is wrong (most probably a corrupt tile)
except Exception:
import sys
e = sys.exc_info()[1]
self._printErrorMessage(e, lzxy)
# remove the status tile
self._mapTiles.removeImageFromMemory(lzxy)
error = constants.TILE_DOWNLOAD_ERROR
finally:
# done, unregister the tile from the tracking set
with self._runningLock:
try:
self._running.remove((lzxy, tag))
except KeyError:
pass
# TODO: find why this happens (well, it appears to be harmless)
# and maybe forward it to debug log once we have one ?
#print("auto tile dl pool: warning, tuple already removed from tracking!")
#print(lzxy)
# report that tha tile has or has not bee successfully downloaded
self._tileDownloaded(error, lzxy, tag)
else:
# don't download tile and remove
# any "downloading" tiles that might
# be in the image cache
self._mapTiles.removeImageFromMemory(lzxy)
# report the tile as not been downloaded
self._tileDownloaded(error, lzxy, tag)
def _downloadTile(self, lzxy):
"""Downloads a tile image image from network"""
self._downloadInProgress(lzxy)
content = self._mapTiles._downloadTile(lzxy)
if content is None:
raise urllib3.exceptions.HTTPError
# cache the raw data
self._mapTiles.storeInMemory(content, lzxy)
self._storeTiles.store_tile_data(lzxy, content)
def _downloadInProgress(self, lzxy):
if self._imageSurface:
# change the status tile to "Downloading..."
self._mapTiles.storeInMemory(self._mapTiles.downloadingTile[0], lzxy, imageType="downloading")
def _temporaryDownloadError(self, lzxy):
if self._imageSurface:
tileNetworkErrorSurface = self._mapTiles.images[1]['tileNetworkError'][0]
expireTimestamp = time.time() + 10
self._mapTiles.storeInMemory(tileNetworkErrorSurface, lzxy, 'error',
expireTimestamp) # retry after 10 seconds
# as not to DOS the system when we temporarily loose internet connection or other such error
# occurs, we load a temporary error tile with expiration timestamp instead of the tile image
# TODO: actually remove tiles according to expiration timestamp :)
return constants.TILE_DOWNLOAD_TEMPORARY_ERROR
def _fatalDownloadError(self, lzxy):
if self._imageSurface:
tileDownloadFailedSurface = self._mapTiles.images[1]['tileDownloadFailed'][0]
expireTimestamp = time.time() + 10
self._mapTiles.storeInMemory(tileDownloadFailedSurface, lzxy, 'semiPermanentError',
expireTimestamp)
# like this, when tile download fails due to a http error,
# the error tile is loaded instead
# like this:
# - modRana does not immediately try to download a tile that errors out
# - the error tile is shown without modifying the pipeline too much
# - modRana will eventually try to download the tile again,
# after it is flushed with old tiles from the memory
return constants.TILE_DOWNLOAD_ERROR
def _printErrorMessage(self, e, lzxy):
url = tiles.getTileUrl(lzxy)
error = "mapTiles: download thread reports error\n"
error+= "** we were doing this, when an exception occurred:\n"
error+= "** downloading tile: x:%d,y:%d,z:%d, layer:%s, url: %s" % (
lzxy[1],
lzxy[2],
lzxy[3],
lzxy[0].id,
url)
log.exception(error)
@property
def maxThreads(self):
return self._pool.maxThreads
@property
def qsize(self):
return self._pool.qsize()