class SaveManager(QObject): start_save = pyqtSignal() report_error = pyqtSignal(object) save_done = pyqtSignal() def __init__(self, parent, notify=None): QObject.__init__(self, parent) self.count = 0 self.last_saved = -1 self.requests = LifoQueue() self.notify_requests = LifoQueue() self.notify_data = notify t = Thread(name='save-thread', target=self.run) t.daemon = True t.start() t = Thread(name='notify-thread', target=self.notify_calibre) t.daemon = True t.start() self.status_widget = w = SaveWidget(parent) self.start_save.connect(w.start, type=Qt.QueuedConnection) self.save_done.connect(w.stop, type=Qt.QueuedConnection) def schedule(self, tdir, container): self.count += 1 self.requests.put((self.count, tdir, container)) def run(self): while True: x = self.requests.get() if x is None: self.requests.task_done() self.__empty_queue() break try: count, tdir, container = x self.process_save(count, tdir, container) except: import traceback traceback.print_exc() finally: self.requests.task_done() def notify_calibre(self): while True: if not self.notify_requests.get(): break send_message(self.notify_data) def clear_notify_data(self): self.notify_data = None def __empty_queue(self): ' Only to be used during shutdown ' while True: try: self.requests.get_nowait() except Empty: break else: self.requests.task_done() def process_save(self, count, tdir, container): if count <= self.last_saved: shutil.rmtree(tdir, ignore_errors=True) return self.last_saved = count self.start_save.emit() try: self.do_save(tdir, container) except: import traceback self.report_error.emit(traceback.format_exc()) self.save_done.emit() if self.notify_data: self.notify_requests.put(True) def do_save(self, tdir, container): try: save_container(container, container.path_to_ebook) finally: shutil.rmtree(tdir, ignore_errors=True) @property def has_tasks(self): return bool(self.requests.unfinished_tasks) def wait(self, timeout=30): if timeout is None: self.requests.join() else: try: join_with_timeout(self.requests, timeout) except RuntimeError: return False return True def shutdown(self): self.requests.put(None) self.notify_requests.put(None)
class SaveManager(QObject): start_save = pyqtSignal() report_error = pyqtSignal(object) save_done = pyqtSignal() def __init__(self, parent): QObject.__init__(self, parent) self.count = 0 self.last_saved = -1 self.requests = LifoQueue() t = Thread(name='save-thread', target=self.run) t.daemon = True t.start() self.status_widget = w = SaveWidget(parent) self.start_save.connect(w.start, type=Qt.QueuedConnection) self.save_done.connect(w.stop, type=Qt.QueuedConnection) def schedule(self, tdir, container): self.count += 1 self.requests.put((self.count, tdir, container)) def run(self): while True: x = self.requests.get() if x is None: self.requests.task_done() self.__empty_queue() break try: count, tdir, container = x self.process_save(count, tdir, container) except: import traceback traceback.print_exc() finally: self.requests.task_done() def __empty_queue(self): ' Only to be used during shutdown ' while True: try: self.requests.get_nowait() except Empty: break else: self.requests.task_done() def process_save(self, count, tdir, container): if count <= self.last_saved: shutil.rmtree(tdir, ignore_errors=True) return self.last_saved = count self.start_save.emit() try: self.do_save(tdir, container) except: import traceback self.report_error.emit(traceback.format_exc()) self.save_done.emit() def do_save(self, tdir, container): try: save_container(container, container.path_to_ebook) finally: shutil.rmtree(tdir, ignore_errors=True) @property def has_tasks(self): return bool(self.requests.unfinished_tasks) def wait(self, timeout=30): if timeout is None: self.requests.join() else: try: join_with_timeout(self.requests, timeout) except RuntimeError: return False return True def shutdown(self): self.requests.put(None)
class TileProvider( QObject ): THREAD_HEARTBEAT = 0.2 Tile = collections.namedtuple('Tile', 'id qimg rectF progress tiling') changed = pyqtSignal( QRectF ) '''TileProvider __init__ Keyword Arguments: cache_size -- maximal number of encountered stacks to cache, i.e. slices if the imagesources draw from slicesources (default 10) request_queue_size -- maximal number of request to queue up (default 100000) n_threads -- maximal number of request threads; this determines the maximal number of simultaneously running requests to the pixelpipeline (default: 2) layerIdChange_means_dirty -- layerId changes invalidate the cache; by default only stackId changes do that (default False) parent -- QObject ''' def __init__( self, tiling, stackedImageSources, cache_size = 10, request_queue_size = 100000, n_threads = 2, layerIdChange_means_dirty=False, parent=None ): QObject.__init__( self, parent = parent ) self.tiling = tiling self._sims = stackedImageSources self._cache_size = cache_size self._request_queue_size = request_queue_size self._n_threads = n_threads self._layerIdChange_means_dirty = layerIdChange_means_dirty self._current_stack_id = self._sims.stackId self._cache = _TilesCache(self._current_stack_id, self._sims, maxstacks=self._cache_size) self._dirtyLayerQueue = LifoQueue(self._request_queue_size) self._sims.layerDirty.connect(self._onLayerDirty) self._sims.visibleChanged.connect(self._onVisibleChanged) self._sims.opacityChanged.connect(self._onOpacityChanged) self._sims.sizeChanged.connect(self._onSizeChanged) self._sims.orderChanged.connect(self._onOrderChanged) self._sims.stackIdChanged.connect(self._onStackIdChanged) if self._layerIdChange_means_dirty: self._sims.layerIdChanged.connect(self._onLayerIdChanged) self._keepRendering = True self._dirtyLayerThreads = [Thread(target=self._dirtyLayersWorker) for i in range(self._n_threads)] for thread in self._dirtyLayerThreads: thread.daemon = True [ thread.start() for thread in self._dirtyLayerThreads ] def getTiles( self, rectF ): '''Get tiles in rect and request a refresh. Returns tiles intersectinf with rectF immediatelly and requests a refresh of these tiles. Next time you call this function the tiles may be already (partially) updated. If you want to wait until the rendering is fully complete, call join(). ''' self.requestRefresh( rectF ) tile_nos = self.tiling.intersectedF( rectF ) stack_id = self._current_stack_id for tile_no in tile_nos: qimg, progress = self._cache.tile(stack_id, tile_no) t = TileProvider.Tile(tile_no, qimg, QRectF(self.tiling.imageRects[tile_no]), progress, self.tiling) yield t def requestRefresh( self, rectF ): '''Requests tiles to be refreshed. Returns immediatelly. Call join() to wait for the end of the rendering. ''' tile_nos = self.tiling.intersectedF( rectF ) for tile_no in tile_nos: stack_id = self._current_stack_id self._refreshTile( stack_id, tile_no ) def join( self ): '''Wait until all refresh request are processed. Blocks until no refresh request pending anymore and all rendering finished. ''' return self._dirtyLayerQueue.join() def notifyThreadsToStop( self ): '''Signals render threads to stop. Call this method at the end of the lifetime of a TileProvider instance. Otherwise the garbage collector will not clean up the instance (even if you call del). ''' self._keepRendering = False def threadsAreNotifiedToStop( self ): '''Check if NotifyThreadsToStop() was called at least once.''' return not self._keepRendering def joinThreads( self, timeout=None ): '''Wait until all threads terminated. Without calling notifyThreadsToStop, threads will never terminate. Arguments: timeout -- timeout in seconds as a floating point number ''' for thread in self._dirtyLayerThreads: thread.join( timeout ) def aliveThreads( self ): '''Return a map of thread identifiers and their alive status. All threads are alive until notifyThreadsToStop() is called. After that, they start dying. Call joinThreads() to wait for the last thread to die. ''' at = {} for thread in self._dirtyLayerThreads: if thread.ident: at[thread.ident] = thread.isAlive() return at def _dirtyLayersWorker( self ): while self._keepRendering: try: ims, tile_nr, stack_id, image_req, timestamp, cache = self._dirtyLayerQueue.get(True, self.THREAD_HEARTBEAT) except (Empty, TypeError): #the TypeError occurs when the self._dirtyLayerQueue #is already None when the thread is being shut down #on program exit. #This avoids a lot of warnings. continue try: if timestamp > cache.layerTimestamp( stack_id, ims, tile_nr ): img = image_req.wait() cache.updateTileIfNecessary( stack_id, ims, tile_nr, timestamp, img ) if stack_id == self._current_stack_id and cache is self._cache: self.changed.emit(QRectF(self.tiling.imageRects[tile_nr])) except KeyError: pass finally: self._dirtyLayerQueue.task_done() def _refreshTile( self, stack_id, tile_no ): try: if self._cache.tileDirty( stack_id, tile_no ): self._cache.setTileDirty(stack_id, tile_no, False) img = self._renderTile( stack_id, tile_no ) self._cache.setTile( stack_id, tile_no, img, self._sims.viewVisible(), self._sims.viewOccluded() ) # refresh dirty layer tiles for ims in self._sims.viewImageSources(): if self._cache.layerDirty(stack_id, ims, tile_no) and not self._sims.isOccluded(ims) and self._sims.isVisible(ims): req = (ims, tile_no, stack_id, ims.request(self.tiling.imageRects[tile_no]), time.time(), self._cache) try: self._dirtyLayerQueue.put_nowait( req ) except Full: warnings.warn("Request queue full. Dropping tile refresh request. Increase queue size!") except KeyError: pass def _renderTile( self, stack_id, tile_nr ): qimg = QImage(self.tiling.imageRects[tile_nr].size(), QImage.Format_ARGB32_Premultiplied) qimg.fill(Qt.white) p = QPainter(qimg) for i, v in enumerate(reversed(self._sims)): visible, layerOpacity, layerImageSource = v if not visible: continue patch = self._cache.layer(stack_id, layerImageSource, tile_nr ) if patch is not None: p.setOpacity(layerOpacity) p.drawImage(0,0, patch) p.end() return qimg def _onLayerDirty(self, dirtyImgSrc, rect ): if dirtyImgSrc in self._sims.viewImageSources(): visibleAndNotOccluded = self._sims.isVisible( dirtyImgSrc ) and not self._sims.isOccluded( dirtyImgSrc ) for tile_no in xrange(len(self.tiling)): #and invalid rect means everything is dirty if not rect.isValid() or self.tiling.tileRects[tile_no].intersected( rect ): for ims in self._sims.viewImageSources(): self._cache.setLayerDirtyAll(ims, tile_no, True) if visibleAndNotOccluded: self._cache.setTileDirtyAll(tile_no, True) if visibleAndNotOccluded: self.changed.emit( QRectF(rect) ) def _onStackIdChanged( self, oldId, newId ): if newId in self._cache: self._cache.touchStack( newId ) else: self._cache.addStack( newId ) self._current_stack_id = newId self.changed.emit(QRectF()) def _onLayerIdChanged( self, ims, oldId, newId ): if self._layerIdChange_means_dirty: self._onLayerDirty( ims, QRect() ) def _onVisibleChanged(self, ims, visible): for tile_no in xrange(len(self.tiling)): self._cache.setTileDirtyAll(tile_no, True) if not self._sims.isOccluded( ims ): self.changed.emit(QRectF()) def _onOpacityChanged(self, ims, opacity): for tile_no in xrange(len(self.tiling)): self._cache.setTileDirtyAll(tile_no, True) if self._sims.isVisible( ims ) and not self._sims.isOccluded( ims ): self.changed.emit(QRectF()) def _onSizeChanged(self): self._cache = _TilesCache(self._current_stack_id, self._sims, maxstacks=self._cache_size) self._dirtyLayerQueue = LifoQueue(self._request_queue_size) self.changed.emit(QRectF()) def _onOrderChanged(self): for tile_no in xrange(len(self.tiling)): self._cache.setTileDirtyAll(tile_no, True) self.changed.emit(QRectF())
class B2BucketThreadedLocal(B2Bucket): def __init__(self, *args): super(B2BucketThreaded, self).__init__(*args) num_threads = 50 self.queue = LifoQueue(num_threads * 2) self.file_locks = defaultdict(Lock) self.running = True self.threads = [] print "Thread ", for i in xrange(num_threads): t = threading.Thread(target=self._file_updater) t.start() self.threads.append(t) print ".", print self.pre_queue_lock = Lock() self.pre_queue_running = True self.pre_queue = LifoQueue(num_threads * 2) self.pre_file_dict = {} self.pre_thread = threading.Thread(target=self._prepare_update) self.pre_thread.start() def _prepare_update(self): while self.pre_queue_running: try: filename, local_filename, operation = self.pre_queue.get( True, 1) self.pre_file_dict[filename] = (time(), local_filename, operation) self.pre_queue.task_done() except Empty: for filename, (timestamp, local_filename, operation) in self.pre_file_dict.items(): if time() - timestamp > 15: self.queue.put((filename, local_filename, operation)) del self.pre_file_dict[filename] for filename, (timestamp, local_filename, operation) in self.pre_file_dict.items(): self.queue.put((filename, local_filename, operation)) del self.pre_file_dict[filename] def _file_updater(self): while self.running: try: filename, local_filename, operation = self.queue.get(True, 1) except Empty: continue with self.file_locks[filename]: if operation == "deletion": super(B2BucketThreaded, self)._delete_file(filename) self.queue.task_done() elif operation == "upload": super(B2BucketThreaded, self)._put_file(filename, local_filename) self.queue.task_done() elif operation == "download": super(B2BucketThreaded, self)._get_file(filename, local_filename) self.queue.task_done() else: self.logger.error("Invalid operation %s on %s" % (operation, filename)) def __enter__(self): return self def __exit__(self, *args, **kwargs): self.logger.info("Waiting for all B2 requests to complete") self.logger.info("Pre-Queue contains %s elements", self.pre_queue.qsize()) self.pre_queue.join() self.logger.info("Joining pre queue thread") self.pre_queue_running = False self.pre_thread.join() self.logger.info("Queue contains %s elements", self.queue.qsize()) self.queue.join() self.logger.info("Joining threads") self.running = False for t in self.threads: t.join() def put_file(self, filename, local_filename): with self.pre_queue_lock: self.logger.info("Postponing upload of %s (%s)", filename, len(data)) self.pre_queue.put((filename, local_filename, "upload"), True) new_file = {} new_file['fileName'] = filename new_file['fileId'] = None new_file['uploadTimestamp'] = time() new_file['action'] = 'upload' new_file['contentLength'] = len(data) return new_file def delete_file(self, filename): with self.pre_queue_lock: self.logger.info("Postponing deletion of %s", filename) self.pre_queue.put((filename, None, "deletion"), True) def get_file(self, filename, local_filename): with self.pre_queue_lock: self.logger.info("Postponing download of %s", filename) self.pre_queue.put((filename, local_filename, "download"), True)
class Imagery: # Number of simultaneous connections to imagery provider and concurrent placement layouts. # Limit to 1 to prevent worker threads slowing UI too much. connections = 1 def __init__(self, canvas): self.providers = { 'Bing': self.bing_setup, 'ArcGIS': self.arcgis_setup, 'MapQuest': self.mq_setup } self.canvas = canvas self.imageryprovider = None self.provider_base = None self.provider_url = None self.provider_logo = None # (filename, width, height) self.provider_levelmin = self.provider_levelmax = 0 self.placementcache = { } # previously created placements (or None if image couldn't be loaded), indexed by quadkey. # placement may not be laid out if image is still being fetched. self.tile = (0, 999) # X-Plane 1x1degree tile - [lat,lon] of SW self.loc = None self.dist = 0 self.filecache = Filecache() # Setup a pool of worker threads self.workers = [] self.q = LifoQueue() for i in range(Imagery.connections): t = threading.Thread(target=self.worker) t.daemon = True # this doesn't appear to work for threads blocked on Queue t.start() self.workers.append(t) # Worker thread def worker(self): # Each thread gets its own tessellators in thread-local storage tls = threading.local() tls.tess = gluNewTess() gluTessNormal(tls.tess, 0, -1, 0) gluTessProperty(tls.tess, GLU_TESS_WINDING_RULE, GLU_TESS_WINDING_NEGATIVE) gluTessCallback(tls.tess, GLU_TESS_VERTEX_DATA, ElevationMeshBase.tessvertex) gluTessCallback(tls.tess, GLU_TESS_EDGE_FLAG, ElevationMeshBase.tessedge) tls.csgt = gluNewTess() gluTessNormal(tls.csgt, 0, -1, 0) gluTessProperty(tls.csgt, GLU_TESS_WINDING_RULE, GLU_TESS_WINDING_ABS_GEQ_TWO) gluTessCallback(tls.csgt, GLU_TESS_VERTEX_DATA, ElevationMeshBase.tessvertex) gluTessCallback(tls.csgt, GLU_TESS_COMBINE, ElevationMeshBase.tesscombinetris) gluTessCallback(tls.csgt, GLU_TESS_EDGE_FLAG, ElevationMeshBase.tessedge) while True: (fn, args) = self.q.get() if not fn: exit() # Die! fn(tls, *args) self.q.task_done() def exit(self): # Closing down self.filecache.writedir() # kill workers for i in range(Imagery.connections): self.q.put( (None, ())) # Top priority! Don't want to hold up program exit # wait for them for t in self.workers: t.join() def reset(self): # Called on reload or on new tile. Empty the cache and forget allocations since: # a) in the case of reload, textures have been dropped and so would need to be reloaded anyway; # b) try to limit cluttering up the VBO with allocations we may not need again; # c) images by straddle multiple tiles and these would need to be recalculated anyway. for placement in self.placementcache.itervalues(): if placement: placement.clearlayout() self.placementcache = {} def goto(self, imageryprovider, loc, dist, screensize): if not imageryprovider: imageryprovider = None if imageryprovider != self.imageryprovider: self.provider_base = None self.provider_url = None self.provider_logo = None self.imageryprovider = imageryprovider if self.imageryprovider not in self.providers: return self.q.put((self.providers[self.imageryprovider], ())) newtile = (int(floor(loc[0])), int(floor(loc[1]))) if not self.provider_url or self.tile != newtile: # New tile - drop cache of Clutter for placement in self.placementcache.itervalues(): if placement: placement.clearlayout() self.placementcache = {} self.tile = newtile self.loc = loc self.placements(dist, screensize) # Kick off any image loading # Return placements to be drawn. May allocate into vertexcache as a side effect. def placements(self, dist, screensize): level0mpp = 2 * pi * 6378137 / 256 # metres per pixel at level 0 if screensize.width <= 0 or not (int(self.loc[0]) or int( self.loc[1])) or not self.provider_url: return [ ] # Don't do anything on startup. Can't do anything without a valid provider. # layout assumes mesh loaded assert ( int(floor(self.loc[0])), int(floor(self.loc[1])), prefs.options & Prefs.ELEVATION ) in self.canvas.vertexcache.elevation, '%s %s' % ( (int(floor(self.loc[0])), int(floor(self.loc[1])), prefs.options & Prefs.ELEVATION), self.canvas.vertexcache.elevation.keys()) # http://msdn.microsoft.com/en-us/library/bb259689.aspx # http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Resolution_and_Scale width = dist + dist # Width in m of screen (from glOrtho setup) ppm = screensize.width / width # Pixels on screen required by 1 metre, ignoring tilt level = min( int(round(log(ppm * level0mpp * cos(radians(self.loc[0])), 2))), self.provider_levelmax) # zoom level required levelmin = max( 13, self.provider_levelmin ) # arbitrary - tessellating out at higher levels takes too long level = max(level, levelmin + 1) ntiles = 2**level # number of tiles per axis at this level #mpp=cos(radians(self.loc[0]))*level0mpp/ntiles # actual resolution at this level #coverage=width/(256*mpp) # how many tiles to cover screen width - in practice varies between ~2.4 and ~4.8 (cx, cy) = self.latlon2xy(self.loc[0], self.loc[1], level) # centre tile #print self.loc, width, screensize.width, 1/ppm, ppm, level, ntiles, cx, cy if __debug__: print "Desire imagery level", level # We're using a Lifo queue, so as the user navigates the most important tiles are processed first. # Should remove from the queue those tiles the user is probably not going to see again, but that's difficult so we don't. # Display 6x6 tiles if available that cover the same area as 3x3 at the next higher level (this is to prevent weirdness when zooming in) cx = 2 * (cx / 2) cy = 2 * (cy / 2) placements = [] needed = set( ) # Placements at this level failed either cos imagery not available at this location/level or is pending layout fetch = [] seq = [(-2, 3), (-1, 3), (0, 3), (1, 3), (2, 3), (3, 3), (3, 2), (3, 1), (3, 0), (3, -1), (3, -2), (2, -2), (1, -2), (0, -2), (-1, -2), (-2, -2), (-2, -1), (-2, 0), (-2, 1), (-2, 2), (-1, 2), (0, 2), (1, 2), (2, 2), (2, 1), (2, 0), (2, -1), (1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1), (1, 0), (0, 0)] for i in range(len(seq)): (x, y) = seq[i] placement = self.getplacement(cx + x, cy + y, level, False) # Don't initiate fetch yet if placement and placement.islaidout(): placements.append(placement) else: needed.add(((cx + x) / 2, (cy + y) / 2)) if 0 <= x <= 1 and 0 <= y <= 1: fetch.append( (cx + x, cy + y, level, True)) # schedule fetch of the centre 2x2 tiles # Go up and get 5x5 tiles around the centre tile - but only draw them if higher-res imagery not (yet) available. level -= 1 cx /= 2 cy /= 2 fail2 = True if self.q.empty(): # If the queue is empty then the first (and low importance) tile starts processing immediately. # So here we add the most important centre tile of 5x5 and ensure it starts processing. placement = self.getplacement(cx, cy, level, True) # Initiate fetch if placement: fail2 = False # Some imagery may be available at this level if placement.islaidout() and (cx, cy) in needed: placements.insert( 0, placement ) # Insert at start so drawn under higher-level needed.remove((cx, cy)) while not self.q.empty(): time.sleep(0) # Allow worker thread to remove from queue # First initiate fetch of higher-level imagery of centre 2x2 for args in fetch: self.getplacement(*args) seq = [(1, -2), (0, -2), (-1, -2), (-2, -1), (-2, 0), (-2, 1), (-1, 2), (0, 2), (1, 2), (2, 1), (2, 0), (2, -1), (1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1), (1, 0), (0, 0) ] # 5x5 with corners removed #, (0,3), (3,0), (0,-3), (-3,0)] for i in range(len(seq)): (x, y) = seq[i] placement = self.getplacement(cx + x, cy + y, level, True) # Initiate fetch if placement: fail2 = False # Some imagery may be available at this level if placement.islaidout() and (abs(x) > 1 or abs(y) > 1 or (cx + x, cy + y) in needed): placements.insert( 0, placement ) # Insert at start so drawn under higher-level while fail2 and level > levelmin: # No imagery available at all at higher level. Go up and get 3x3 tiles around the centre tile. level -= 1 cx /= 2 cy /= 2 seq = [(1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1), (1, 0), (0, 0)] for i in range(len(seq)): (x, y) = seq[i] placement = self.getplacement(cx + x, cy + y, level, True) if placement: fail2 = False # Some imagery may be available at this level if placement.islaidout(): placements.insert( 0, placement ) # Insert at start so drawn under higher-level if __debug__: print "Actual imagery level", level return placements # Helper to return coordinates in a spiral from http://stackoverflow.com/questions/398299/looping-in-a-spiral def spiral(self, N, M): x = y = 0 dx, dy = 0, -1 retval = [] for dumb in xrange(N * M): if abs(x) == abs(y) and [dx, dy] != [1, 0] or x > 0 and y == 1 - x: dx, dy = -dy, dx # corner, change direction if abs(x) > N / 2 or abs(y) > M / 2: # non-square dx, dy = -dy, dx # change direction x, y = -y + dx, x + dy # jump retval.append((x, y)) x, y = x + dx, y + dy return retval # Returns a laid-out placement if possible, or not laid-out if image is still loading, or None if not available. def getplacement(self, x, y, level, fetch): (name, url) = self.provider_url(x, y, level) if name in self.placementcache: # Already created placement = self.placementcache[name] elif fetch: # Make a new one. We could also do this if the image file is available, but don't since layout is expensive. (north, west) = self.xy2latlon(x, y, level) (south, east) = self.xy2latlon(x + 1, y + 1, level) placement = DrapedImage(name, 65535, [[ Node([west, north, 0, 1]), Node([east, north, 1, 1]), Node([east, south, 1, 0]), Node([west, south, 0, 0]) ]]) placement.load(self.canvas.lookup, self.canvas.defs, self.canvas.vertexcache) self.placementcache[name] = placement # Initiate fetch of image and do layout. Prioritise more detail. self.q.put((self.initplacement, (placement, name, url))) else: placement = None # Load it if it's not loaded but is ready to be if placement and not placement.islaidout() and Polygon.islaidout( placement): try: if __debug__: clock = time.clock() filename = self.filecache.get(name) # downloaded image or None self.canvas.vertexcache.allocate_dynamic( placement, True) # couldn't do this in thread context placement.definition.texture = self.canvas.vertexcache.texcache.get( filename, wrap=False, downsample=False, fixsize=True) if __debug__: print "%6.3f time in imagery load for %s" % ( time.clock() - clock, placement.name) assert placement.islaidout() except: if __debug__: print_exc() # Some failure - perhaps corrupted image? placement.clearlayout() placement = None self.placementcache[name] = None return placement def latlon2xy(self, lat, lon, level): ntiles = 2**level # number of tiles per axis at this level sinlat = sin(radians(lat)) x = int((lon + 180) * ntiles / 360.0) y = int((0.5 - (log((1 + sinlat) / (1 - sinlat)) / fourpi)) * ntiles) return (x, y) def xy2latlon(self, x, y, level): ntiles = float(2**level) # number of tiles per axis at this level lat = 90 - 360 * atan(exp((y / ntiles - 0.5) * 2 * pi)) / pi lon = x * 360.0 / ntiles - 180 return (lat, lon) def bing_quadkey(self, x, y, level): # http://msdn.microsoft.com/en-us/library/bb259689.aspx i = level quadkey = '' while i > 0: digit = 0 mask = 1 << (i - 1) if (x & mask): digit += 1 if (y & mask): digit += 2 quadkey += ('%d' % digit) i -= 1 url = self.provider_base % quadkey name = basename(url).split('?')[0] return (name, url) # Called in worker thread - don't do anything fancy since main body of code is not thread-safe def bing_setup(self, tls): try: key = 'AhATjCXv4Sb-i_YKsa_8lF4DtHwVoicFxl0Stc9QiXZNywFbI2rajKZCsLFIMOX2' h = urlopen( 'http://dev.virtualearth.net/REST/v1/Imagery/Metadata/Aerial?key=%s' % key) d = h.read() h.close() info = json_decode(d) # http://msdn.microsoft.com/en-us/library/ff701707.aspx if 'authenticationResultCode' not in info or info[ 'authenticationResultCode'] != 'ValidCredentials' or 'statusCode' not in info or info[ 'statusCode'] != 200: return res = info['resourceSets'][0]['resources'][0] # http://msdn.microsoft.com/en-us/library/ff701712.aspx self.provider_levelmin = int(res['zoomMin']) self.provider_levelmax = int(res['zoomMax']) self.provider_base = res['imageUrl'].replace( '{subdomain}', res['imageUrlSubdomains'][-1] ).replace('{culture}', 'en').replace( '{quadkey}', '%s' ) + '&key=' + key # was random.choice(res['imageUrlSubdomains']) but always picking the same server seems to give better caching self.provider_url = self.bing_quadkey if info['brandLogoUri']: filename = self.filecache.fetch(basename(info['brandLogoUri']), info['brandLogoUri']) if filename: image = PIL.Image.open( filename) # yuck. but at least open is lazy self.provider_logo = (filename, image.size[0], image.size[1]) except: if __debug__: print_exc() self.canvas.Refresh() # Might have been waiting on this to get imagery def arcgis_url(self, x, y, level): url = self.provider_base % ("%d/%d/%d" % (level, y, x)) name = "arcgis_%d_%d_%d.jpeg" % (level, y, x) return (name, url) # Called in worker thread - don't do anything fancy since main body of code is not thread-safe def arcgis_setup(self, tls): try: h = urlopen( 'http://services.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer?f=json' ) d = h.read() h.close() info = json_decode(d) # http://resources.arcgis.com/en/help/rest/apiref/index.html # http://resources.arcgis.com/en/help/rest/apiref/mapserver.html self.provider_levelmin = min( [lod['level'] for lod in info['tileInfo']['lods']]) self.provider_levelmax = max( [lod['level'] for lod in info['tileInfo']['lods']]) self.provider_base = 'http://services.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/%s' self.provider_url = self.arcgis_url filename = self.filecache.fetch( 'logo-med.png', 'http://serverapi.arcgisonline.com/jsapi/arcgis/2.8/images/map/logo-med.png' ) if filename: image = PIL.Image.open( filename) # yuck. but at least open is lazy self.provider_logo = (filename, image.size[0], image.size[1]) except: if __debug__: print_exc() self.canvas.Refresh() # Might have been waiting on this to get imagery def mq_url(self, x, y, level): url = self.provider_base % ("%d/%d/%d.jpg" % (level, x, y)) name = "mq_%d_%d_%d.jpg" % (level, x, y) return (name, url) # Called in worker thread - don't do anything fancy since main body of code is not thread-safe def mq_setup(self, tls): # http://developer.mapquest.com/web/products/open/map try: self.provider_levelmin = 0 self.provider_levelmax = 18 self.provider_base = 'http://otile1.mqcdn.com/tiles/1.0.0/map/%s' self.provider_url = self.mq_url filename = self.filecache.fetch( 'questy.png', 'http://open.mapquest.com/cdn/toolkit/lite/images/questy.png') if filename: image = PIL.Image.open( filename) # yuck. but at least open is lazy self.provider_logo = (filename, image.size[0], image.size[1]) except: if __debug__: print_exc() self.canvas.Refresh() # Might have been waiting on this to get imagery # Called in worker thread - fetches image and does placement layout (which uses it's own tessellator and so is thread-safe). # don't do anything fancy since main body of code is not thread-safe def initplacement(self, tls, placement, name, url): filename = self.filecache.fetch(name, url) if not filename: # Couldn't fetch image - remove corresponding placement self.placementcache[name] = None else: if __debug__: clock = time.clock() placement.layout(self.tile, tls=tls) if not placement.dynamic_data.size: if __debug__: print "DrapedImage layout failed for %s - no tris" % placement.name self.placementcache[name] = None elif __debug__: print "%6.3f time in imagery layout for %s" % ( time.clock() - clock, placement.name) self.canvas.Refresh( ) # Probably wanting to display this - corresponding placement will be loaded and laid out during OnPaint
class ThreadPool(object): def __init__(self, threadNum, max_tasks_per_period=10, seconds_per_period=30): self.pool = [] # 线程池 self.threadNum = threadNum # 线程数 self.runningLock = Lock() # 线程锁 self.taskLock = Lock() # getTask函数的锁 self.running = 0 # 正在run的线程数 # 设置为LIFO队列:在抓取了第一个post的页面后,随后需要添加所有其后的评论页, # 使用LIFO队列可以保证尽快将第一个post的所有评论抓取到,并存储 self.taskQueue = LifoQueue() # 任务队列 # 一分钟内允许的最大访问次数 self.max_tasks_per_period = max_tasks_per_period # 定制每分钟含有的秒数 self.seconds_per_period = seconds_per_period # 当前周期内已经访问的网页数量 self.currentPeriodVisits = 0 # 将一分钟当作一个访问周期,记录当前周期的开始时间 self.periodStart = time.time() # 使用当前时间初始化 def startThreads(self): """Create a certain number of threads and started to run All Workers share the same ThreadPool """ # 开始当前的抓取周期 self.periodStart = time.time() for i in range(self.threadNum): self.pool.append(Worker(self, i)) def stopThreads(self): for thread in self.pool: thread.stop() thread.join() del self.pool[:] def putTask(self, func, *args, **kargs): self.taskQueue.put((func, args, kargs)) def getTask(self, *args, **kargs): # 进行访问控制: 判断当前周期内访问的网页数目是否大于最大数目 if self.currentPeriodVisits >= self.max_tasks_per_period - 2: timeNow = time.time() seconds = timeNow - self.periodStart if seconds < self.seconds_per_period: # 如果当前还没有过一分钟,则sleep remain = self.seconds_per_period - seconds print "ThreadPool Waiting for " + str(remain) + " seconds." time.sleep(int(remain + 1)) self.periodStart = time.time() # 重新设置开始时间 self.currentPeriodVisits = 0 try: # task = self.taskQueue.get(*args, **kargs) task = self.taskQueue.get_nowait() except Empty: return (None, None, None) self.currentPeriodVisits += 1 return task def taskJoin(self, *args, **kargs): """Queue.join: Blocks until all items in the queue have been gotten and processed. """ self.taskQueue.join() def taskDone(self, *args, **kargs): self.taskQueue.task_done() def increaseRunsNum(self): self.runningLock.acquire() self.running += 1 # 正在运行的线程数加1 self.runningLock.release() def decreaseRunsNum(self): self.runningLock.acquire() self.running -= 1 self.runningLock.release() def getTaskLeft(self): # 线程池的所有任务包括: # taskQueue中未被下载的任务, resultQueue中完成了但是还没被取出的任务, 正在运行的任务 # 因此任务总数为三者之和 return self.taskQueue.qsize() + self.running
class Imagery: # Number of simultaneous connections to imagery provider and concurrent placement layouts. # Limit to 1 to prevent worker threads slowing UI too much. connections=1 def __init__(self, canvas): self.providers={'Bing': self.bing_setup, 'ArcGIS': self.arcgis_setup, 'MapQuest': self.mq_setup } self.canvas=canvas self.imageryprovider=None self.provider_base=None self.provider_url=None self.provider_logo=None # (filename, width, height) self.provider_levelmin=self.provider_levelmax=0 self.placementcache={} # previously created placements (or None if image couldn't be loaded), indexed by quadkey. # placement may not be laid out if image is still being fetched. self.tile=(0,999) # X-Plane 1x1degree tile - [lat,lon] of SW self.loc=None self.dist=0 self.filecache=Filecache() # Setup a pool of worker threads self.workers=[] self.q=LifoQueue() for i in range(Imagery.connections): t=threading.Thread(target=self.worker) t.daemon=True # this doesn't appear to work for threads blocked on Queue t.start() self.workers.append(t) # Worker thread def worker(self): # Each thread gets its own tessellators in thread-local storage tls=threading.local() tls.tess=gluNewTess() gluTessNormal(tls.tess, 0, -1, 0) gluTessProperty(tls.tess, GLU_TESS_WINDING_RULE, GLU_TESS_WINDING_NEGATIVE) gluTessCallback(tls.tess, GLU_TESS_VERTEX_DATA, ElevationMeshBase.tessvertex) gluTessCallback(tls.tess, GLU_TESS_EDGE_FLAG, ElevationMeshBase.tessedge) tls.csgt=gluNewTess() gluTessNormal(tls.csgt, 0, -1, 0) gluTessProperty(tls.csgt, GLU_TESS_WINDING_RULE, GLU_TESS_WINDING_ABS_GEQ_TWO) gluTessCallback(tls.csgt, GLU_TESS_VERTEX_DATA, ElevationMeshBase.tessvertex) gluTessCallback(tls.csgt, GLU_TESS_COMBINE, ElevationMeshBase.tesscombinetris) gluTessCallback(tls.csgt, GLU_TESS_EDGE_FLAG, ElevationMeshBase.tessedge) while True: (fn, args)=self.q.get() if not fn: exit() # Die! fn(tls, *args) self.q.task_done() def exit(self): # Closing down self.filecache.writedir() # kill workers for i in range(Imagery.connections): self.q.put((None, ())) # Top priority! Don't want to hold up program exit # wait for them for t in self.workers: t.join() def reset(self): # Called on reload or on new tile. Empty the cache and forget allocations since: # a) in the case of reload, textures have been dropped and so would need to be reloaded anyway; # b) try to limit cluttering up the VBO with allocations we may not need again; # c) images by straddle multiple tiles and these would need to be recalculated anyway. for placement in self.placementcache.itervalues(): if placement: placement.clearlayout() self.placementcache={} def goto(self, imageryprovider, loc, dist, screensize): if not imageryprovider: imageryprovider=None if imageryprovider!=self.imageryprovider: self.provider_base=None self.provider_url=None self.provider_logo=None self.imageryprovider=imageryprovider if self.imageryprovider not in self.providers: return self.q.put((self.providers[self.imageryprovider], ())) newtile=(int(floor(loc[0])),int(floor(loc[1]))) if not self.provider_url or self.tile!=newtile: # New tile - drop cache of Clutter for placement in self.placementcache.itervalues(): if placement: placement.clearlayout() self.placementcache={} self.tile=newtile self.loc=loc self.placements(dist, screensize) # Kick off any image loading # Return placements to be drawn. May allocate into vertexcache as a side effect. def placements(self, dist, screensize): level0mpp=2*pi*6378137/256 # metres per pixel at level 0 if screensize.width<=0 or not (int(self.loc[0]) or int(self.loc[1])) or not self.provider_url: return [] # Don't do anything on startup. Can't do anything without a valid provider. # layout assumes mesh loaded assert (int(floor(self.loc[0])),int(floor(self.loc[1])),prefs.options&Prefs.ELEVATION) in self.canvas.vertexcache.elevation, '%s %s' % ((int(floor(self.loc[0])),int(floor(self.loc[1])),prefs.options&Prefs.ELEVATION), self.canvas.vertexcache.elevation.keys()) # http://msdn.microsoft.com/en-us/library/bb259689.aspx # http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Resolution_and_Scale width=dist+dist # Width in m of screen (from glOrtho setup) ppm=screensize.width/width # Pixels on screen required by 1 metre, ignoring tilt level=min(int(round(log(ppm*level0mpp*cos(radians(self.loc[0])), 2))), self.provider_levelmax) # zoom level required levelmin=max(13, self.provider_levelmin) # arbitrary - tessellating out at higher levels takes too long level=max(level,levelmin+1) ntiles=2**level # number of tiles per axis at this level #mpp=cos(radians(self.loc[0]))*level0mpp/ntiles # actual resolution at this level #coverage=width/(256*mpp) # how many tiles to cover screen width - in practice varies between ~2.4 and ~4.8 (cx,cy)=self.latlon2xy(self.loc[0], self.loc[1], level) # centre tile #print self.loc, width, screensize.width, 1/ppm, ppm, level, ntiles, cx, cy if __debug__: print "Desire imagery level", level # We're using a Lifo queue, so as the user navigates the most important tiles are processed first. # Should remove from the queue those tiles the user is probably not going to see again, but that's difficult so we don't. # Display 6x6 tiles if available that cover the same area as 3x3 at the next higher level (this is to prevent weirdness when zooming in) cx=2*(cx/2) cy=2*(cy/2) placements=[] needed=set() # Placements at this level failed either cos imagery not available at this location/level or is pending layout fetch=[] seq=[(-2, 3), (-1, 3), (0, 3), (1, 3), (2, 3), (3, 3), (3, 2), (3, 1), (3, 0), (3, -1), (3, -2), (2, -2), (1, -2), (0, -2), (-1, -2), (-2, -2), (-2, -1), (-2, 0), (-2, 1), (-2, 2), (-1, 2), (0, 2), (1, 2), (2, 2), (2, 1), (2, 0), (2, -1), (1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1), (1, 0), (0, 0)] for i in range(len(seq)): (x,y)=seq[i] placement=self.getplacement(cx+x,cy+y,level, False) # Don't initiate fetch yet if placement and placement.islaidout(): placements.append(placement) else: needed.add(((cx+x)/2,(cy+y)/2)) if 0<=x<=1 and 0<=y<=1: fetch.append((cx+x,cy+y,level, True)) # schedule fetch of the centre 2x2 tiles # Go up and get 5x5 tiles around the centre tile - but only draw them if higher-res imagery not (yet) available. level-=1 cx/=2 cy/=2 fail2=True if self.q.empty(): # If the queue is empty then the first (and low importance) tile starts processing immediately. # So here we add the most important centre tile of 5x5 and ensure it starts processing. placement=self.getplacement(cx,cy,level, True) # Initiate fetch if placement: fail2=False # Some imagery may be available at this level if placement.islaidout() and (cx,cy) in needed: placements.insert(0,placement) # Insert at start so drawn under higher-level needed.remove((cx,cy)) while not self.q.empty(): time.sleep(0) # Allow worker thread to remove from queue # First initiate fetch of higher-level imagery of centre 2x2 for args in fetch: self.getplacement(*args) seq=[(1, -2), (0, -2), (-1, -2), (-2, -1), (-2, 0), (-2, 1), (-1, 2), (0, 2), (1, 2), (2, 1), (2, 0), (2, -1), (1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1), (1, 0), (0, 0)] # 5x5 with corners removed #, (0,3), (3,0), (0,-3), (-3,0)] for i in range(len(seq)): (x,y)=seq[i] placement=self.getplacement(cx+x,cy+y,level, True) # Initiate fetch if placement: fail2=False # Some imagery may be available at this level if placement.islaidout() and (abs(x)>1 or abs(y)>1 or (cx+x,cy+y) in needed): placements.insert(0,placement) # Insert at start so drawn under higher-level while fail2 and level>levelmin: # No imagery available at all at higher level. Go up and get 3x3 tiles around the centre tile. level-=1 cx/=2 cy/=2 seq=[(1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1), (1, 0), (0, 0)] for i in range(len(seq)): (x,y)=seq[i] placement=self.getplacement(cx+x,cy+y,level, True) if placement: fail2=False # Some imagery may be available at this level if placement.islaidout(): placements.insert(0,placement) # Insert at start so drawn under higher-level if __debug__: print "Actual imagery level", level return placements # Helper to return coordinates in a spiral from http://stackoverflow.com/questions/398299/looping-in-a-spiral def spiral(self, N, M): x = y = 0 dx, dy = 0, -1 retval=[] for dumb in xrange(N*M): if abs(x) == abs(y) and [dx,dy] != [1,0] or x>0 and y == 1-x: dx, dy = -dy, dx # corner, change direction if abs(x)>N/2 or abs(y)>M/2: # non-square dx, dy = -dy, dx # change direction x, y = -y+dx, x+dy # jump retval.append((x,y)) x, y = x+dx, y+dy return retval # Returns a laid-out placement if possible, or not laid-out if image is still loading, or None if not available. def getplacement(self,x,y,level,fetch): (name,url)=self.provider_url(x,y,level) if name in self.placementcache: # Already created placement=self.placementcache[name] elif fetch: # Make a new one. We could also do this if the image file is available, but don't since layout is expensive. (north,west)=self.xy2latlon(x,y,level) (south,east)=self.xy2latlon(x+1,y+1,level) placement=DrapedImage(name, 65535, [[Node([west,north,0,1]),Node([east,north,1,1]),Node([east,south,1,0]),Node([west,south,0,0])]]) placement.load(self.canvas.lookup, self.canvas.defs, self.canvas.vertexcache) self.placementcache[name]=placement # Initiate fetch of image and do layout. Prioritise more detail. self.q.put((self.initplacement, (placement,name,url))) else: placement=None # Load it if it's not loaded but is ready to be if placement and not placement.islaidout() and Polygon.islaidout(placement): try: if __debug__: clock=time.clock() filename=self.filecache.get(name) # downloaded image or None self.canvas.vertexcache.allocate_dynamic(placement, True) # couldn't do this in thread context placement.definition.texture=self.canvas.vertexcache.texcache.get(filename, wrap=False, downsample=False, fixsize=True) if __debug__: print "%6.3f time in imagery load for %s" % (time.clock()-clock, placement.name) assert placement.islaidout() except: if __debug__: print_exc() # Some failure - perhaps corrupted image? placement.clearlayout() placement=None self.placementcache[name]=None return placement def latlon2xy(self, lat, lon, level): ntiles=2**level # number of tiles per axis at this level sinlat=sin(radians(lat)) x = int((lon+180) * ntiles/360.0) y = int( (0.5 - (log((1+sinlat)/(1-sinlat)) / fourpi)) * ntiles) return (x,y) def xy2latlon(self, x, y, level): ntiles=float(2**level) # number of tiles per axis at this level lat = 90 - 360 * atan(exp((y/ntiles-0.5)*2*pi)) / pi lon = x*360.0/ntiles - 180 return (lat,lon) def bing_quadkey(self, x, y, level): # http://msdn.microsoft.com/en-us/library/bb259689.aspx i=level quadkey='' while i>0: digit=0 mask = 1 << (i-1) if (x & mask): digit+=1 if (y & mask): digit+=2 quadkey+=('%d' % digit) i-=1 url=self.provider_base % quadkey name=basename(url).split('?')[0] return (name,url) # Called in worker thread - don't do anything fancy since main body of code is not thread-safe def bing_setup(self, tls): try: key='AhATjCXv4Sb-i_YKsa_8lF4DtHwVoicFxl0Stc9QiXZNywFbI2rajKZCsLFIMOX2' h=urlopen('http://dev.virtualearth.net/REST/v1/Imagery/Metadata/Aerial?key=%s' % key) d=h.read() h.close() info=json_decode(d) # http://msdn.microsoft.com/en-us/library/ff701707.aspx if 'authenticationResultCode' not in info or info['authenticationResultCode']!='ValidCredentials' or 'statusCode' not in info or info['statusCode']!=200: return res=info['resourceSets'][0]['resources'][0] # http://msdn.microsoft.com/en-us/library/ff701712.aspx self.provider_levelmin=int(res['zoomMin']) self.provider_levelmax=int(res['zoomMax']) self.provider_base=res['imageUrl'].replace('{subdomain}',res['imageUrlSubdomains'][-1]).replace('{culture}','en').replace('{quadkey}','%s') + '&key=' + key # was random.choice(res['imageUrlSubdomains']) but always picking the same server seems to give better caching self.provider_url=self.bing_quadkey if info['brandLogoUri']: filename=self.filecache.fetch(basename(info['brandLogoUri']), info['brandLogoUri']) if filename: image = PIL.Image.open(filename) # yuck. but at least open is lazy self.provider_logo=(filename,image.size[0],image.size[1]) except: if __debug__: print_exc() self.canvas.Refresh() # Might have been waiting on this to get imagery def arcgis_url(self, x, y, level): url=self.provider_base % ("%d/%d/%d" % (level, y, x)) name="arcgis_%d_%d_%d.jpeg" % (level, y, x) return (name,url) # Called in worker thread - don't do anything fancy since main body of code is not thread-safe def arcgis_setup(self, tls): try: h=urlopen('http://services.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer?f=json') d=h.read() h.close() info=json_decode(d) # http://resources.arcgis.com/en/help/rest/apiref/index.html # http://resources.arcgis.com/en/help/rest/apiref/mapserver.html self.provider_levelmin=min([lod['level'] for lod in info['tileInfo']['lods']]) self.provider_levelmax=max([lod['level'] for lod in info['tileInfo']['lods']]) self.provider_base='http://services.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/%s' self.provider_url=self.arcgis_url filename=self.filecache.fetch('logo-med.png', 'http://serverapi.arcgisonline.com/jsapi/arcgis/2.8/images/map/logo-med.png') if filename: image = PIL.Image.open(filename) # yuck. but at least open is lazy self.provider_logo=(filename,image.size[0],image.size[1]) except: if __debug__: print_exc() self.canvas.Refresh() # Might have been waiting on this to get imagery def mq_url(self, x, y, level): url=self.provider_base % ("%d/%d/%d.jpg" % (level, x, y)) name="mq_%d_%d_%d.jpg" % (level, x, y) return (name,url) # Called in worker thread - don't do anything fancy since main body of code is not thread-safe def mq_setup(self, tls): # http://developer.mapquest.com/web/products/open/map try: self.provider_levelmin=0 self.provider_levelmax=18 self.provider_base='http://otile1.mqcdn.com/tiles/1.0.0/map/%s' self.provider_url=self.mq_url filename=self.filecache.fetch('questy.png', 'http://open.mapquest.com/cdn/toolkit/lite/images/questy.png') if filename: image = PIL.Image.open(filename) # yuck. but at least open is lazy self.provider_logo=(filename,image.size[0],image.size[1]) except: if __debug__: print_exc() self.canvas.Refresh() # Might have been waiting on this to get imagery # Called in worker thread - fetches image and does placement layout (which uses it's own tessellator and so is thread-safe). # don't do anything fancy since main body of code is not thread-safe def initplacement(self, tls, placement, name, url): filename=self.filecache.fetch(name, url) if not filename: # Couldn't fetch image - remove corresponding placement self.placementcache[name]=None else: if __debug__: clock=time.clock() placement.layout(self.tile, tls=tls) if not placement.dynamic_data.size: if __debug__: print "DrapedImage layout failed for %s - no tris" % placement.name self.placementcache[name]=None elif __debug__: print "%6.3f time in imagery layout for %s" % (time.clock()-clock, placement.name) self.canvas.Refresh() # Probably wanting to display this - corresponding placement will be loaded and laid out during OnPaint
class Transceiver(object): def __init__(self, config={}, message_handler=None): """Set up a receiver which connects to the messaging hub. :param config: This is a dict in the form:: config = dict( incoming='tcp://localhost:15566', # default outgoing='tcp://localhost:15567', idle_timeout=1000, # milliseconds: ) """ self.log = logging.getLogger("evasion.messenger.endpoint.Transceiver") self.endpoint_uuid = str(uuid.uuid4()) self.exit_time = threading.Event() self.wait_for_exit = threading.Event() self.incoming = None # configured in main(). self.incoming_uri = config.get("incoming", 'tcp://localhost:15566') self.log.info("Recieving on <%s>" % self.incoming_uri) self.outgoing_uri = config.get("outgoing", 'tcp://localhost:15567') self.log.info("Sending on <%s>" % self.outgoing_uri) self.idle_timeout = int(config.get("idle_timeout", 2000)) self.log.info("Idle Timeout (ms): %d" % self.idle_timeout) self.message_handler = message_handler self.sync_message = frames.sync_message( "endpoint-%s" % self.endpoint_uuid ) # Queue up messages to be sent in the main message loop self._out_queue = LifoQueue() def main(self): """Running the main loop sending and receiving. This will keep running until stop() is called. This sets the exit flag causing clean up and shutdown. """ self.exitTime = False context = zmq.Context() incoming = context.socket(zmq.SUB) incoming.setsockopt(zmq.SUBSCRIBE, '') incoming.connect(self.incoming_uri) outgoing = context.socket(zmq.PUSH); outgoing.connect(self.outgoing_uri); def _shutdown(): try: incoming.close() except ZMQError: self.log.exception("main: error calling incoming.close()") try: outgoing.close() except ZMQError: self.log.exception("main: error calling outgoing.close()") try: context.term() except ZMQError: self.log.exception("main: error calling context.term()") try: poller = zmq.Poller() poller.register(incoming, zmq.POLLIN) while not self.exit_time.is_set(): try: events = poller.poll(self.idle_timeout) except ZMQError as e: # 4 = 'Interrupted system call' if e.errno == 4: self.log.info("main: exit time: %s" % e) break else: self.log.info("main: <%s>" % e) break except Exception: self.log.exception("main: fatal error while polling ") break else: if (events > 0): msg = incoming.recv_multipart() self.message_in(tuple(msg)) # Now recover and queued outgoing messages: if not self._out_queue.empty(): message = self._out_queue.get_nowait() if message: try: # send sync hub followed by message. The sync # will kick the hub into life if its just # started: outgoing.send_multipart(self.sync_message) outgoing.send_multipart(message) except ZMQError as e: # 4 = 'Interrupted system call' if e.errno == 4: self.log.info(( "main: sigint or other signal interrupt" ", exit time <%s>" ) % e) break else: self.log.info("main: <%s>" % e) break except Exception: self.log.exception("main: fatal error sending ") break finally: self._out_queue.task_done() finally: self.wait_for_exit.set() _shutdown() def start(self): """Set up zmq communication and start receiving messages from the hub. """ # coverage can't seem to get to this: def _main(notused): # pragma: no cover self.exit_time.clear() self.wait_for_exit.clear() self.main() thread.start_new(_main, (0,)) def stop(self, wait=2): """Stop receiving messages from the hub and clean up. :param wait: The time in seconds to wait before giving up on a clean shutdown. """ self.log.info("stop: shutting down messaging.") self.exit_time.set() self.wait_for_exit.wait(wait) self.log.info("stop: done.") def message_out(self, message): """This sends a message to the messagehub for dispatch to all connected endpoints. :param message: A tuple or list representing a multipart ZMQ message. If the message is not a tuple or list then MessageOutError will be raised. Note: The message is actually queued here so that the main loop will send it when its ready. :returns: None. """ if isinstance(message, list) or isinstance(message, tuple): self._out_queue.put(message) else: m = "The message must be a list or tuple instead of <%s>" % type( message ) raise MessageOutError(m) def message_in(self, message): """Called on receipt of an evasion frame to determine what to do. The message_handler set in the constructer will be called if one was set. If none was set then the message will be logged at the DEBUG level. :param message: A tuple or list representing a multipart ZMQ message. :returns: None. """ if self.message_handler: try: #self.log.debug("message_in: message <%s>" % str(message)) self.message_handler(message) except: self.log.exception("message_in: Error handling message - ") else: self.log.debug("message_in: message <%s>" % str(message))
class B2BucketThreadedLocal(B2Bucket): def __init__(self, *args): super(B2BucketThreaded, self).__init__( *args) num_threads=50 self.queue = LifoQueue(num_threads*2) self.file_locks = defaultdict(Lock) self.running = True self.threads = [] print "Thread ", for i in xrange(num_threads): t = threading.Thread(target=self._file_updater) t.start() self.threads.append(t) print ".", print self.pre_queue_lock = Lock() self.pre_queue_running = True self.pre_queue = LifoQueue(num_threads*2) self.pre_file_dict = {} self.pre_thread = threading.Thread(target=self._prepare_update) self.pre_thread.start() def _prepare_update(self): while self.pre_queue_running: try: filename, local_filename, operation = self.pre_queue.get(True,1) self.pre_file_dict[filename] = (time(), local_filename, operation) self.pre_queue.task_done() except Empty: for filename, (timestamp, local_filename, operation) in self.pre_file_dict.items(): if time()-timestamp > 15: self.queue.put((filename, local_filename, operation)) del self.pre_file_dict[filename] for filename, (timestamp, local_filename, operation) in self.pre_file_dict.items(): self.queue.put((filename, local_filename, operation)) del self.pre_file_dict[filename] def _file_updater(self): while self.running: try: filename, local_filename, operation = self.queue.get(True,1) except Empty: continue with self.file_locks[filename]: if operation == "deletion": super(B2BucketThreaded,self)._delete_file(filename) self.queue.task_done() elif operation == "upload": super(B2BucketThreaded,self)._put_file(filename, local_filename) self.queue.task_done() elif operation == "download": super(B2BucketThreaded,self)._get_file(filename, local_filename) self.queue.task_done() else: self.logger.error("Invalid operation %s on %s" % (operation, filename)) def __enter__(self): return self def __exit__(self, *args, **kwargs): self.logger.info("Waiting for all B2 requests to complete") self.logger.info("Pre-Queue contains %s elements", self.pre_queue.qsize()) self.pre_queue.join() self.logger.info("Joining pre queue thread") self.pre_queue_running = False self.pre_thread.join() self.logger.info("Queue contains %s elements", self.queue.qsize()) self.queue.join() self.logger.info("Joining threads") self.running = False for t in self.threads: t.join() def put_file(self, filename, local_filename): with self.pre_queue_lock: self.logger.info("Postponing upload of %s (%s)", filename, len(data)) self.pre_queue.put((filename, local_filename, "upload"), True) new_file = {} new_file['fileName'] = filename new_file['fileId'] = None new_file['uploadTimestamp'] = time() new_file['action'] = 'upload' new_file['contentLength'] = len(data) return new_file def delete_file(self, filename): with self.pre_queue_lock: self.logger.info("Postponing deletion of %s", filename) self.pre_queue.put((filename, None, "deletion"),True) def get_file(self, filename, local_filename): with self.pre_queue_lock: self.logger.info("Postponing download of %s", filename) self.pre_queue.put((filename, local_filename, "download"),True)
class SaveManager(QObject): start_save = pyqtSignal() report_error = pyqtSignal(object) save_done = pyqtSignal() def __init__(self, parent): QObject.__init__(self, parent) self.count = 0 self.last_saved = -1 self.requests = LifoQueue() t = Thread(name='save-thread', target=self.run) t.daemon = True t.start() self.status_widget = w = SaveWidget(parent) self.start_save.connect(w.start, type=Qt.QueuedConnection) self.save_done.connect(w.stop, type=Qt.QueuedConnection) def schedule(self, tdir, container): self.count += 1 self.requests.put((self.count, tdir, container)) def run(self): while True: x = self.requests.get() if x is None: self.requests.task_done() self.__empty_queue() break try: count, tdir, container = x self.process_save(count, tdir, container) except: import traceback traceback.print_exc() finally: self.requests.task_done() def __empty_queue(self): ' Only to be used during shutdown ' while True: try: self.requests.get_nowait() except Empty: break else: self.requests.task_done() def process_save(self, count, tdir, container): if count <= self.last_saved: shutil.rmtree(tdir, ignore_errors=True) return self.last_saved = count self.start_save.emit() try: self.do_save(tdir, container) except: import traceback self.report_error.emit(traceback.format_exc()) self.save_done.emit() def do_save(self, tdir, container): temp = None try: path = container.path_to_ebook temp = PersistentTemporaryFile(prefix=('_' if iswindows else '.'), suffix=os.path.splitext(path)[1], dir=os.path.dirname(path)) temp.close() temp = temp.name container.commit(temp) atomic_rename(temp, path) finally: if temp and os.path.exists(temp): os.remove(temp) shutil.rmtree(tdir, ignore_errors=True) @property def has_tasks(self): return bool(self.requests.unfinished_tasks) def wait(self, timeout=30): if timeout is None: self.requests.join() else: try: join_with_timeout(self.requests, timeout) except RuntimeError: return False return True def shutdown(self): self.requests.put(None)