class DownloadService(object):
def __init__(self, process, sess=None):
self.sess = sess
def _process(meta, status, data):
self.cache_tile(meta['tile'], status, data, meta.get('cache', False), meta.get('overwrite', False))
process(meta, status, data)
return (True, None)
self.dlmgr = DownloadManager(limit=100)
self.dlpxr = DownloadProcessor(self.dlmgr, _process)
self.started = False
def add(self, key, url):
if not self.started:
self.start()
self.dlmgr.enqueue((key, url))
def cache_tile(self, t, status, data, cache, overwrite):
if cache:
if overwrite or not self.sess.query(mt.Tile).get(t.pk()):
try:
_process_tile(self.sess, t, status, data)
except:
pass
def start(self):
self.started = True
self.dlmgr.start()
self.dlpxr.start()
def terminate(self):
self.dlmgr.terminate()
self.dlpxr.terminate()
class TileDownloader(threading.Thread):
"""a monitorable thread that downloads and processes tiles"""
def __init__(self, tiles, sess):
threading.Thread.__init__(self)
self.tiles = tiles
self.num_tiles = len(tiles)
# error count and last error are displayed in the curses interface
self.error_count = 0
self.last_error = None
self.dlmgr = DownloadManager(limit=100)
def process(key, status, data):
return process_tile(sess, key, status, data)
self.dlpxr = DownloadProcessor(self.dlmgr, process, self.num_tiles, self.onerror)
def run(self):
self.dlmgr.start()
self.dlpxr.start()
for t in random_walk(self.tiles):
layer, z, x, y = t
tile = mt.Tile(layer=layer, z=z, x=x, y=y)
self.dlmgr.enqueue((tile, tile.url()))
self.dlpxr.join()
self.dlmgr.terminate()
self.dlmgr.join()
def onerror(self, msg):
self.error_count += 1
self.last_error = msg
def status(self):
return (self.dlpxr.count, self.num_tiles, self.error_count)
