class DownloadService(object): def __init__(self, process, sess=None): self.sess = sess def _process(meta, status, data): self.cache_tile(meta['tile'], status, data, meta.get('cache', False), meta.get('overwrite', False)) process(meta, status, data) return (True, None) self.dlmgr = DownloadManager(limit=100) self.dlpxr = DownloadProcessor(self.dlmgr, _process) self.started = False def add(self, key, url): if not self.started: self.start() self.dlmgr.enqueue((key, url)) def cache_tile(self, t, status, data, cache, overwrite): if cache: if overwrite or not self.sess.query(mt.Tile).get(t.pk()): try: _process_tile(self.sess, t, status, data) except: pass def start(self): self.started = True self.dlmgr.start() self.dlpxr.start() def terminate(self): self.dlmgr.terminate() self.dlpxr.terminate()
class TileDownloader(threading.Thread): """a monitorable thread that downloads and processes tiles""" def __init__(self, tiles, sess): threading.Thread.__init__(self) self.tiles = tiles self.num_tiles = len(tiles) # error count and last error are displayed in the curses interface self.error_count = 0 self.last_error = None self.dlmgr = DownloadManager(limit=100) def process(key, status, data): return process_tile(sess, key, status, data) self.dlpxr = DownloadProcessor(self.dlmgr, process, self.num_tiles, self.onerror) def run(self): self.dlmgr.start() self.dlpxr.start() for t in random_walk(self.tiles): layer, z, x, y = t tile = mt.Tile(layer=layer, z=z, x=x, y=y) self.dlmgr.enqueue((tile, tile.url())) self.dlpxr.join() self.dlmgr.terminate() self.dlmgr.join() def onerror(self, msg): self.error_count += 1 self.last_error = msg def status(self): return (self.dlpxr.count, self.num_tiles, self.error_count)