class ConnectionPool(object): def __init__( self, connection_class=Connection, max_connections=1024, timeout=5, **connection_kwargs): self.connection_class = connection_class self.max_connections = max_connections self.connection_kwargs = connection_kwargs self.timeout = timeout self.connections = [] self.reset() def ensure_safe(self): if self.pid != os.getpid(): with self.ensure_lock: # lock for concurrent threadings if self.pid == os.getpid(): # double check return self.disconnect() self.reset() def get_connection(self): self.ensure_safe() try: connection = self.queue.get(block=True, timeout=self.timeout) except Empty: raise ConnectionError('connection pool is full') if not connection: connection = self.make_connection() return connection def make_connection(self): connection = self.connection_class(**self.connection_kwargs) self.connections.append(connection) return connection def release(self, connection): self.ensure_safe() if connection.pid != self.pid: return try: self.queue.put_nowait(connection) except Full: pass def reset(self): self.pid = os.getpid() self.ensure_lock = threading.Lock() self.disconnect() # LifoQueue make use of released connections self.queue = LifoQueue(self.max_connections) self.connections = [] while True: try: self.queue.put_nowait(None) except Full: break def disconnect(self): for connection in self.connections: connection.disconnect()
def DFS(N, L, root): queue = LifoQueue() queue.put_nowait(root) while queue.qsize() != 0: arrangement = queue.get_nowait() if arrangement.L == L: return arrangement.Map else: search(N, arrangement, queue) return 0
class TileProvider( QObject ): THREAD_HEARTBEAT = 0.2 Tile = collections.namedtuple('Tile', 'id qimg rectF progress tiling') changed = pyqtSignal( QRectF ) '''TileProvider __init__ Keyword Arguments: cache_size -- maximal number of encountered stacks to cache, i.e. slices if the imagesources draw from slicesources (default 10) request_queue_size -- maximal number of request to queue up (default 100000) n_threads -- maximal number of request threads; this determines the maximal number of simultaneously running requests to the pixelpipeline (default: 2) layerIdChange_means_dirty -- layerId changes invalidate the cache; by default only stackId changes do that (default False) parent -- QObject ''' def __init__( self, tiling, stackedImageSources, cache_size = 10, request_queue_size = 100000, n_threads = 2, layerIdChange_means_dirty=False, parent=None ): QObject.__init__( self, parent = parent ) self.tiling = tiling self._sims = stackedImageSources self._cache_size = cache_size self._request_queue_size = request_queue_size self._n_threads = n_threads self._layerIdChange_means_dirty = layerIdChange_means_dirty self._current_stack_id = self._sims.stackId self._cache = _TilesCache(self._current_stack_id, self._sims, maxstacks=self._cache_size) self._dirtyLayerQueue = LifoQueue(self._request_queue_size) self._sims.layerDirty.connect(self._onLayerDirty) self._sims.visibleChanged.connect(self._onVisibleChanged) self._sims.opacityChanged.connect(self._onOpacityChanged) self._sims.sizeChanged.connect(self._onSizeChanged) self._sims.orderChanged.connect(self._onOrderChanged) self._sims.stackIdChanged.connect(self._onStackIdChanged) if self._layerIdChange_means_dirty: self._sims.layerIdChanged.connect(self._onLayerIdChanged) self._keepRendering = True self._dirtyLayerThreads = [Thread(target=self._dirtyLayersWorker) for i in range(self._n_threads)] for thread in self._dirtyLayerThreads: thread.daemon = True [ thread.start() for thread in self._dirtyLayerThreads ] def getTiles( self, rectF ): '''Get tiles in rect and request a refresh. Returns tiles intersectinf with rectF immediatelly and requests a refresh of these tiles. Next time you call this function the tiles may be already (partially) updated. If you want to wait until the rendering is fully complete, call join(). ''' self.requestRefresh( rectF ) tile_nos = self.tiling.intersectedF( rectF ) stack_id = self._current_stack_id for tile_no in tile_nos: qimg, progress = self._cache.tile(stack_id, tile_no) t = TileProvider.Tile(tile_no, qimg, QRectF(self.tiling.imageRects[tile_no]), progress, self.tiling) yield t def requestRefresh( self, rectF ): '''Requests tiles to be refreshed. Returns immediatelly. Call join() to wait for the end of the rendering. ''' tile_nos = self.tiling.intersectedF( rectF ) for tile_no in tile_nos: stack_id = self._current_stack_id self._refreshTile( stack_id, tile_no ) def join( self ): '''Wait until all refresh request are processed. Blocks until no refresh request pending anymore and all rendering finished. ''' return self._dirtyLayerQueue.join() def notifyThreadsToStop( self ): '''Signals render threads to stop. Call this method at the end of the lifetime of a TileProvider instance. Otherwise the garbage collector will not clean up the instance (even if you call del). ''' self._keepRendering = False def threadsAreNotifiedToStop( self ): '''Check if NotifyThreadsToStop() was called at least once.''' return not self._keepRendering def joinThreads( self, timeout=None ): '''Wait until all threads terminated. Without calling notifyThreadsToStop, threads will never terminate. Arguments: timeout -- timeout in seconds as a floating point number ''' for thread in self._dirtyLayerThreads: thread.join( timeout ) def aliveThreads( self ): '''Return a map of thread identifiers and their alive status. All threads are alive until notifyThreadsToStop() is called. After that, they start dying. Call joinThreads() to wait for the last thread to die. ''' at = {} for thread in self._dirtyLayerThreads: if thread.ident: at[thread.ident] = thread.isAlive() return at def _dirtyLayersWorker( self ): while self._keepRendering: try: ims, tile_nr, stack_id, image_req, timestamp, cache = self._dirtyLayerQueue.get(True, self.THREAD_HEARTBEAT) except (Empty, TypeError): #the TypeError occurs when the self._dirtyLayerQueue #is already None when the thread is being shut down #on program exit. #This avoids a lot of warnings. continue try: if timestamp > cache.layerTimestamp( stack_id, ims, tile_nr ): img = image_req.wait() cache.updateTileIfNecessary( stack_id, ims, tile_nr, timestamp, img ) if stack_id == self._current_stack_id and cache is self._cache: self.changed.emit(QRectF(self.tiling.imageRects[tile_nr])) except KeyError: pass finally: self._dirtyLayerQueue.task_done() def _refreshTile( self, stack_id, tile_no ): try: if self._cache.tileDirty( stack_id, tile_no ): self._cache.setTileDirty(stack_id, tile_no, False) img = self._renderTile( stack_id, tile_no ) self._cache.setTile( stack_id, tile_no, img, self._sims.viewVisible(), self._sims.viewOccluded() ) # refresh dirty layer tiles for ims in self._sims.viewImageSources(): if self._cache.layerDirty(stack_id, ims, tile_no) and not self._sims.isOccluded(ims) and self._sims.isVisible(ims): req = (ims, tile_no, stack_id, ims.request(self.tiling.imageRects[tile_no]), time.time(), self._cache) try: self._dirtyLayerQueue.put_nowait( req ) except Full: warnings.warn("Request queue full. Dropping tile refresh request. Increase queue size!") except KeyError: pass def _renderTile( self, stack_id, tile_nr ): qimg = QImage(self.tiling.imageRects[tile_nr].size(), QImage.Format_ARGB32_Premultiplied) qimg.fill(Qt.white) p = QPainter(qimg) for i, v in enumerate(reversed(self._sims)): visible, layerOpacity, layerImageSource = v if not visible: continue patch = self._cache.layer(stack_id, layerImageSource, tile_nr ) if patch is not None: p.setOpacity(layerOpacity) p.drawImage(0,0, patch) p.end() return qimg def _onLayerDirty(self, dirtyImgSrc, rect ): if dirtyImgSrc in self._sims.viewImageSources(): visibleAndNotOccluded = self._sims.isVisible( dirtyImgSrc ) and not self._sims.isOccluded( dirtyImgSrc ) for tile_no in xrange(len(self.tiling)): #and invalid rect means everything is dirty if not rect.isValid() or self.tiling.tileRects[tile_no].intersected( rect ): for ims in self._sims.viewImageSources(): self._cache.setLayerDirtyAll(ims, tile_no, True) if visibleAndNotOccluded: self._cache.setTileDirtyAll(tile_no, True) if visibleAndNotOccluded: self.changed.emit( QRectF(rect) ) def _onStackIdChanged( self, oldId, newId ): if newId in self._cache: self._cache.touchStack( newId ) else: self._cache.addStack( newId ) self._current_stack_id = newId self.changed.emit(QRectF()) def _onLayerIdChanged( self, ims, oldId, newId ): if self._layerIdChange_means_dirty: self._onLayerDirty( ims, QRect() ) def _onVisibleChanged(self, ims, visible): for tile_no in xrange(len(self.tiling)): self._cache.setTileDirtyAll(tile_no, True) if not self._sims.isOccluded( ims ): self.changed.emit(QRectF()) def _onOpacityChanged(self, ims, opacity): for tile_no in xrange(len(self.tiling)): self._cache.setTileDirtyAll(tile_no, True) if self._sims.isVisible( ims ) and not self._sims.isOccluded( ims ): self.changed.emit(QRectF()) def _onSizeChanged(self): self._cache = _TilesCache(self._current_stack_id, self._sims, maxstacks=self._cache_size) self._dirtyLayerQueue = LifoQueue(self._request_queue_size) self.changed.emit(QRectF()) def _onOrderChanged(self): for tile_no in xrange(len(self.tiling)): self._cache.setTileDirtyAll(tile_no, True) self.changed.emit(QRectF())
class ConnectionPool(object): """Thread-safe connection pool implementation. ``max_connections`` Maximum number of connections. ``block_timeout`` Maximum number of seconds to wait. """ def __init__(self, max_connections=10, block_timeout=5, **kwds): self.max_connections = max_connections self.block_timeout = block_timeout # kwds is being sent directly to dbapi, so we pop self._pool_id = kwds.pop('pool_id') self._logobj = kwds.pop('logobj') self.kwds = kwds # make sure max_connections is valid is_valid = isinstance(max_connections, int) and \ max_connections > 0 if not is_valid: raise ValueError('max_connections must be a positive int') # if process id is changed, we will close all connections, # and reinstantiate this object self._pid = os.getpid() # this is where we define the pool, and fill it with None values self._pool = LifoQueue(max_connections) while True: try: self._pool.put_nowait(None) except Full: break # open connections self._connections = [] def _checkpid(self): """Closes all connections and reinstantiates the object if pid is changed. """ if self._pid == os.getpid(): return self.disconnect() self.reinstantiate() def _make_connection(self): """Creates a fresh connection. """ connection = psycopg2.connect(**self.kwds) if not hasattr(connection, 'pool_id'): connection.pool_id = self._pool_id # we don't need transaction, so let's turn autocommit on connection.autocommit = True # pass the logger object if needed if isinstance(connection, LoggingConnection): connection.initialize(self._logobj) _register_hstore(connection, unicode=True) # encoding connection.set_client_encoding('UTF8') # timezone cursor = connection.cursor() cursor.execute("set timezone to 'UTC'") cursor.close() self._connections.append(connection) return connection def get_connection(self): """Returns a psycopg2 connection for the given shard_id. Raises `ConnectionError` if necessary. """ self._checkpid() connection = None try: # wait for a connection connection = self._pool.get(block=True, timeout=self.block_timeout) except Empty: # timeout raise ConnectionError('no connection available') # create a new connection if it is not initialized yet if connection is None: connection = self._make_connection() return connection def put_connection(self, connection): """Sends connection back into the pool. """ self._checkpid() try: self._pool.put_nowait(connection) except Full: # reinstantiate may have caused this. # this connection is useless now. pass def disconnect(self): """Closes every connection in every pool. """ for connection in self._connections: connection.close() self._connections = [] def reinstantiate(self): """Reinstantiates connection pools. Make sure you have closed every connection before calling this method. """ # let's add these back to kwds self.kwds['pool_id'] = self._pool_id self.kwds['logobj'] = self._logobj self.__init__(max_connections=self.max_connections, block_timeout=self.block_timeout, **self.kwds)
class SpeechSynthesizer: def __init__(self): # Queue holding the last speech utterance self._speak_queue = LifoQueue(1) self._session = Session(profile_name="mylespolly") self._polly = self._session.client("polly", region_name="eu-west-1") self._thread = Thread(target=self.run, args=()) self._thread.daemon = True self._thread.start() def request(self, text): """Clear queue (ignore it being empty) and add text, both non-blocking""" # for Python 3.4+ this could be written as: with suppress(Empty): ... assuming: from contextlib import suppress try: self._speak_queue.get_nowait() except Empty: pass self._speak_queue.put_nowait(text) def run(self): """Continuously process the queue and trigger speech outputs""" while True: text = self._speak_queue.get(True, None) print(text) try: response = self._polly.synthesize_speech(Text=text, OutputFormat="mp3", VoiceId="Salli") except (BotoCoreError, ClientError) as error: print(error) sys.exit(-1) # Access the audio stream from the response if "AudioStream" in response: # Note: Closing the stream is important as the service throttles on the # number of parallel connections. Here we are using contextlib.closing to # ensure the close method of the stream object will be called automatically # at the end of the with statement's scope. with closing(response["AudioStream"]) as stream: output = os.path.join(gettempdir(), "speech.mp3") print(output) try: # Open a file for writing the output as a binary stream with open(output, "wb") as file: file.write(stream.read()) except IOError as error: # Could not write to file, exit gracefully print(error) sys.exit(-1) else: # The response didn't contain audio data, exit gracefully print("Could not stream audio") sys.exit(-1) # Play the audio using VLC # see https://wiki.videolan.org/Python_bindings # see https://www.olivieraubert.net/vlc/python-ctypes/doc/index.html p = vlc.MediaPlayer(output) sleep(0.1) p.play() sleep(0.1) while p.is_playing(): pass
class TileProvider(QObject): THREAD_HEARTBEAT = 0.2 Tile = collections.namedtuple('Tile', 'id qimg rectF progress tiling') sceneRectChanged = pyqtSignal(QRectF) '''TileProvider __init__ Keyword Arguments: cache_size -- maximal number of encountered stacks to cache, i.e. slices if the imagesources draw from slicesources (default 10) request_queue_size -- maximal number of request to queue up (default 100000) n_threads -- maximal number of request threads; this determines the maximal number of simultaneously running requests to the pixelpipeline (default: 2) layerIdChange_means_dirty -- layerId changes invalidate the cache; by default only stackId changes do that (default False) parent -- QObject ''' @property def axesSwapped(self): return self._axesSwapped @axesSwapped.setter def axesSwapped(self, value): self._axesSwapped = value def __init__(self, tiling, stackedImageSources, cache_size=100, request_queue_size=100000, n_threads=2, layerIdChange_means_dirty=False, parent=None): QObject.__init__(self, parent=parent) self.tiling = tiling self.axesSwapped = False self._sims = stackedImageSources self._cache_size = cache_size self._request_queue_size = request_queue_size self._n_threads = n_threads self._layerIdChange_means_dirty = layerIdChange_means_dirty self._current_stack_id = self._sims.stackId self._cache = _TilesCache(self._current_stack_id, self._sims, maxstacks=self._cache_size) self._dirtyLayerQueue = LifoQueue(self._request_queue_size) self._prefetchQueue = Queue(self._request_queue_size) self._sims.layerDirty.connect(self._onLayerDirty) self._sims.visibleChanged.connect(self._onVisibleChanged) self._sims.opacityChanged.connect(self._onOpacityChanged) self._sims.sizeChanged.connect(self._onSizeChanged) self._sims.orderChanged.connect(self._onOrderChanged) self._sims.stackIdChanged.connect(self._onStackIdChanged) if self._layerIdChange_means_dirty: self._sims.layerIdChanged.connect(self._onLayerIdChanged) self._keepRendering = True self._dirtyLayerThreads = [ Thread(target=self._dirtyLayersWorker) for i in range(self._n_threads) ] for thread in self._dirtyLayerThreads: thread.daemon = True [thread.start() for thread in self._dirtyLayerThreads] def getTiles(self, rectF): '''Get tiles in rect and request a refresh. Returns tiles intersecting with rectF immediately and requests a refresh of these tiles. Next time you call this function the tiles may be already (partially) updated. If you want to wait until the rendering is fully complete, call join(). ''' self.requestRefresh(rectF) tile_nos = self.tiling.intersected(rectF) stack_id = self._current_stack_id for tile_no in tile_nos: qimg, progress = self._cache.tile(stack_id, tile_no) yield TileProvider.Tile(tile_no, qimg, QRectF(self.tiling.imageRects[tile_no]), progress, self.tiling) def requestRefresh(self, rectF): '''Requests tiles to be refreshed. Returns immediately. Call join() to wait for the end of the rendering. ''' tile_nos = self.tiling.intersected(rectF) for tile_no in tile_nos: stack_id = self._current_stack_id self._refreshTile(stack_id, tile_no) def prefetch(self, rectF, through): '''Request fetching of tiles in advance. Returns immediately. Prefetch will commence after all regular tiles are refreshed (see requestRefresh() and getTiles() ). The prefetch is reset when the 'through' value of the slicing changes. Several calls to prefetch are handeled in Fifo order. ''' if self._cache_size > 1: stack_id = (self._current_stack_id[0], enumerate(through)) if stack_id not in self._cache: self._cache.addStack(stack_id) self._cache.touchStack(self._current_stack_id) tile_nos = self.tiling.intersected(rectF) for tile_no in tile_nos: self._refreshTile(stack_id, tile_no, prefetch=True) def join(self): '''Wait until all refresh request are processed. Blocks until no refresh request pending anymore and all rendering finished. ''' return self._dirtyLayerQueue.join() def notifyThreadsToStop(self): '''Signals render threads to stop. Call this method at the end of the lifetime of a TileProvider instance. Otherwise the garbage collector will not clean up the instance (even if you call del). ''' self._keepRendering = False def threadsAreNotifiedToStop(self): '''Check if NotifyThreadsToStop() was called at least once.''' return not self._keepRendering def joinThreads(self, timeout=None): '''Wait until all threads terminated. Without calling notifyThreadsToStop, threads will never terminate. Arguments: timeout -- timeout in seconds as a floating point number ''' for thread in self._dirtyLayerThreads: thread.join(timeout) def aliveThreads(self): '''Return a map of thread identifiers and their alive status. All threads are alive until notifyThreadsToStop() is called. After that, they start dying. Call joinThreads() to wait for the last thread to die. ''' at = {} for thread in self._dirtyLayerThreads: if thread.ident: at[thread.ident] = thread.isAlive() return at def _dirtyLayersWorker(self): while self._keepRendering: # Save reference to the queue in case self._dirtyLayerQueue reassigned during this pass. # See onSizeChanged() dirtyLayerQueue = self._dirtyLayerQueue prefetchQueue = self._prefetchQueue try: try: result = dirtyLayerQueue.get_nowait() queue = dirtyLayerQueue except Empty: try: result = prefetchQueue.get_nowait() queue = prefetchQueue except Empty: try: result = dirtyLayerQueue.get( True, self.THREAD_HEARTBEAT) queue = dirtyLayerQueue except Empty: continue except TypeError: #the TypeError occurs when the queue #is already None when the thread is being shut down #on program exit. #This avoids a lot of warnings. continue ims, transform, tile_nr, stack_id, image_req, timestamp, cache = result try: try: layerTimestamp = cache.layerTimestamp( stack_id, ims, tile_nr) except KeyError: pass else: if timestamp > layerTimestamp: img = image_req.wait() img = img.transformed(transform) try: cache.updateTileIfNecessary( stack_id, ims, tile_nr, timestamp, img) except KeyError: pass else: if stack_id == self._current_stack_id and cache is self._cache: self.sceneRectChanged.emit( QRectF(self.tiling.imageRects[tile_nr])) except: with volumina.printLock: sys.excepthook(*sys.exc_info()) sys.stderr.write( "ERROR: volumina tiling layer rendering worker thread caught an unhandled exception. See above." ) finally: queue.task_done() def _refreshTile(self, stack_id, tile_no, prefetch=False): if not self.axesSwapped: transform = QTransform(0, 1, 0, 1, 0, 0, 1, 1, 1) else: transform = QTransform().rotate(90).scale(1, -1) transform *= self.tiling.data2scene try: if self._cache.tileDirty(stack_id, tile_no): if not prefetch: self._cache.setTileDirty(stack_id, tile_no, False) img = self._renderTile(stack_id, tile_no) self._cache.setTile(stack_id, tile_no, img, self._sims.viewVisible(), self._sims.viewOccluded()) # refresh dirty layer tiles for ims in self._sims.viewImageSources(): if self._cache.layerDirty(stack_id, ims, tile_no) \ and not self._sims.isOccluded(ims) \ and self._sims.isVisible(ims): rect = self.tiling.imageRects[tile_no] dataRect = self.tiling.scene2data.mapRect(rect) ims_req = ims.request(dataRect, stack_id[1]) if ims.direct and not prefetch: # The ImageSource 'ims' is fast (it has the # direct flag set to true) so we process # the request synchronously here. This # improves the responsiveness for layers # that have the data readily available. start = time.time() img = ims_req.wait() img = img.transformed(transform) stop = time.time() ims._layer.timePerTile( stop - start, self.tiling.imageRects[tile_no]) self._cache.updateTileIfNecessary( stack_id, ims, tile_no, time.time(), img) img = self._renderTile(stack_id, tile_no) self._cache.setTile(stack_id, tile_no, img, self._sims.viewVisible(), self._sims.viewOccluded()) else: req = (ims, transform, tile_no, stack_id, ims_req, time.time(), self._cache) try: if prefetch: self._prefetchQueue.put_nowait(req) else: self._dirtyLayerQueue.put_nowait(req) except Full: msg = " ".join( ("Request queue full.", "Dropping tile refresh request.", "Increase queue size!")) warnings.warn(msg) except KeyError: pass def _renderTile(self, stack_id, tile_nr): qimg = None p = None for i, v in enumerate(reversed(self._sims)): visible, layerOpacity, layerImageSource = v if not visible: continue patch = self._cache.layer(stack_id, layerImageSource, tile_nr) if patch is not None: if qimg is None: qimg = QImage(self.tiling.imageRects[tile_nr].size(), QImage.Format_ARGB32_Premultiplied) qimg.fill(0xffffffff) # Use a hex constant instead. p = QPainter(qimg) p.setOpacity(layerOpacity) p.drawImage(0, 0, patch) if p is not None: p.end() return qimg def _onLayerDirty(self, dirtyImgSrc, dataRect): sceneRect = self.tiling.data2scene.mapRect(dataRect) if dirtyImgSrc in self._sims.viewImageSources(): visibleAndNotOccluded = self._sims.isVisible( dirtyImgSrc ) \ and not self._sims.isOccluded( dirtyImgSrc ) for tile_no in xrange(len(self.tiling)): # an invalid rect means everything is dirty if not sceneRect.isValid() \ or self.tiling.tileRects[tile_no].intersected( sceneRect ): for ims in self._sims.viewImageSources(): self._cache.setLayerDirtyAll(ims, tile_no, True) if visibleAndNotOccluded: self._cache.setTileDirtyAll(tile_no, True) if visibleAndNotOccluded: self.sceneRectChanged.emit(QRectF(sceneRect)) def _onStackIdChanged(self, oldId, newId): if newId in self._cache: self._cache.touchStack(newId) else: self._cache.addStack(newId) self._current_stack_id = newId self._prefetchQueue = Queue(self._request_queue_size) self.sceneRectChanged.emit(QRectF()) def _onLayerIdChanged(self, ims, oldId, newId): if self._layerIdChange_means_dirty: self._onLayerDirty(ims, QRect()) def _onVisibleChanged(self, ims, visible): for tile_no in xrange(len(self.tiling)): self._cache.setTileDirtyAll(tile_no, True) if not self._sims.isOccluded(ims): self.sceneRectChanged.emit(QRectF()) def _onOpacityChanged(self, ims, opacity): for tile_no in xrange(len(self.tiling)): self._cache.setTileDirtyAll(tile_no, True) if self._sims.isVisible(ims) and not self._sims.isOccluded(ims): self.sceneRectChanged.emit(QRectF()) def _onSizeChanged(self): self._cache = _TilesCache(self._current_stack_id, self._sims, maxstacks=self._cache_size) self._dirtyLayerQueue = LifoQueue(self._request_queue_size) self._prefetchQueue = Queue(self._request_queue_size) self.sceneRectChanged.emit(QRectF()) def _onOrderChanged(self): for tile_no in xrange(len(self.tiling)): self._cache.setTileDirtyAll(tile_no, True) self.sceneRectChanged.emit(QRectF())
class TileProvider( QObject ): THREAD_HEARTBEAT = 0.2 Tile = collections.namedtuple('Tile', 'id qimg rectF progress tiling') sceneRectChanged = pyqtSignal( QRectF ) '''TileProvider __init__ Keyword Arguments: cache_size -- maximal number of encountered stacks to cache, i.e. slices if the imagesources draw from slicesources (default 10) request_queue_size -- maximal number of request to queue up (default 100000) n_threads -- maximal number of request threads; this determines the maximal number of simultaneously running requests to the pixelpipeline (default: 2) layerIdChange_means_dirty -- layerId changes invalidate the cache; by default only stackId changes do that (default False) parent -- QObject ''' @property def axesSwapped(self): return self._axesSwapped @axesSwapped.setter def axesSwapped(self, value): self._axesSwapped = value def __init__( self, tiling, stackedImageSources, cache_size=100, request_queue_size=100000, n_threads=2, layerIdChange_means_dirty=False, parent=None ): QObject.__init__( self, parent = parent ) self.tiling = tiling self.axesSwapped = False self._sims = stackedImageSources self._cache_size = cache_size self._request_queue_size = request_queue_size self._n_threads = n_threads self._layerIdChange_means_dirty = layerIdChange_means_dirty self._current_stack_id = self._sims.stackId self._cache = _TilesCache(self._current_stack_id, self._sims, maxstacks=self._cache_size) self._dirtyLayerQueue = LifoQueue(self._request_queue_size) self._prefetchQueue = Queue(self._request_queue_size) self._sims.layerDirty.connect(self._onLayerDirty) self._sims.visibleChanged.connect(self._onVisibleChanged) self._sims.opacityChanged.connect(self._onOpacityChanged) self._sims.sizeChanged.connect(self._onSizeChanged) self._sims.orderChanged.connect(self._onOrderChanged) self._sims.stackIdChanged.connect(self._onStackIdChanged) if self._layerIdChange_means_dirty: self._sims.layerIdChanged.connect(self._onLayerIdChanged) self._keepRendering = True self._dirtyLayerThreads = [Thread(target=self._dirtyLayersWorker) for i in range(self._n_threads)] for thread in self._dirtyLayerThreads: thread.daemon = True [ thread.start() for thread in self._dirtyLayerThreads ] def getTiles( self, rectF ): '''Get tiles in rect and request a refresh. Returns tiles intersecting with rectF immediately and requests a refresh of these tiles. Next time you call this function the tiles may be already (partially) updated. If you want to wait until the rendering is fully complete, call join(). ''' self.requestRefresh( rectF ) tile_nos = self.tiling.intersected( rectF ) stack_id = self._current_stack_id for tile_no in tile_nos: qimg, progress = self._cache.tile(stack_id, tile_no) yield TileProvider.Tile( tile_no, qimg, QRectF(self.tiling.imageRects[tile_no]), progress, self.tiling) def requestRefresh( self, rectF ): '''Requests tiles to be refreshed. Returns immediately. Call join() to wait for the end of the rendering. ''' tile_nos = self.tiling.intersected( rectF ) for tile_no in tile_nos: stack_id = self._current_stack_id self._refreshTile( stack_id, tile_no ) def prefetch( self, rectF, through ): '''Request fetching of tiles in advance. Returns immediately. Prefetch will commence after all regular tiles are refreshed (see requestRefresh() and getTiles() ). The prefetch is reset when the 'through' value of the slicing changes. Several calls to prefetch are handeled in Fifo order. ''' if self._cache_size > 1: stack_id = (self._current_stack_id[0], through) if stack_id not in self._cache: self._cache.addStack(stack_id) self._cache.touchStack( self._current_stack_id ) tile_nos = self.tiling.intersected( rectF ) for tile_no in tile_nos: self._refreshTile( stack_id, tile_no, prefetch=True ) def join( self ): '''Wait until all refresh request are processed. Blocks until no refresh request pending anymore and all rendering finished. ''' return self._dirtyLayerQueue.join() def notifyThreadsToStop( self ): '''Signals render threads to stop. Call this method at the end of the lifetime of a TileProvider instance. Otherwise the garbage collector will not clean up the instance (even if you call del). ''' self._keepRendering = False def threadsAreNotifiedToStop( self ): '''Check if NotifyThreadsToStop() was called at least once.''' return not self._keepRendering def joinThreads( self, timeout=None ): '''Wait until all threads terminated. Without calling notifyThreadsToStop, threads will never terminate. Arguments: timeout -- timeout in seconds as a floating point number ''' for thread in self._dirtyLayerThreads: thread.join( timeout ) def aliveThreads( self ): '''Return a map of thread identifiers and their alive status. All threads are alive until notifyThreadsToStop() is called. After that, they start dying. Call joinThreads() to wait for the last thread to die. ''' at = {} for thread in self._dirtyLayerThreads: if thread.ident: at[thread.ident] = thread.isAlive() return at def _dirtyLayersWorker( self ): while self._keepRendering: # Save reference to the queue in case self._dirtyLayerQueue reassigned during this pass. # See onSizeChanged() dirtyLayerQueue = self._dirtyLayerQueue prefetchQueue = self._prefetchQueue try: try: result = dirtyLayerQueue.get_nowait() queue = dirtyLayerQueue except Empty: try: result = prefetchQueue.get_nowait() queue = prefetchQueue except Empty: try: result = dirtyLayerQueue.get(True, self.THREAD_HEARTBEAT) queue = dirtyLayerQueue except Empty: continue except TypeError: #the TypeError occurs when the queue #is already None when the thread is being shut down #on program exit. #This avoids a lot of warnings. continue ims, transform, tile_nr, stack_id, image_req, timestamp, cache = result try: try: layerTimestamp = cache.layerTimestamp( stack_id, ims, tile_nr ) except KeyError: pass else: if timestamp > layerTimestamp: img = image_req.wait() img = img.transformed(transform) try: cache.updateTileIfNecessary( stack_id, ims, tile_nr, timestamp, img ) except KeyError: pass else: if stack_id == self._current_stack_id and cache is self._cache: self.sceneRectChanged.emit(QRectF(self.tiling.imageRects[tile_nr])) except: with volumina.printLock: sys.excepthook( *sys.exc_info() ) sys.stderr.write("ERROR: volumina tiling layer rendering worker thread caught an unhandled exception. See above.") finally: queue.task_done() def _refreshTile( self, stack_id, tile_no, prefetch=False ): if not self.axesSwapped: transform = QTransform(0,1,0,1,0,0,1,1,1) else: transform = QTransform().rotate(90).scale(1,-1) transform *= self.tiling.data2scene try: if self._cache.tileDirty( stack_id, tile_no ): if not prefetch: self._cache.setTileDirty(stack_id, tile_no, False) img = self._renderTile( stack_id, tile_no ) self._cache.setTile(stack_id, tile_no, img, self._sims.viewVisible(), self._sims.viewOccluded()) # refresh dirty layer tiles for ims in self._sims.viewImageSources(): if self._cache.layerDirty(stack_id, ims, tile_no) \ and not self._sims.isOccluded(ims) \ and self._sims.isVisible(ims): rect = self.tiling.imageRects[tile_no] dataRect = self.tiling.scene2data.mapRect(rect) ims_req = ims.request(dataRect, stack_id[1]) if ims.direct: # The ImageSource 'ims' is fast (it has the # direct flag set to true) so we process # the request synchronously here. This # improves the responsiveness for layers # that have the data readily available. start = time.time() img = ims_req.wait() img = img.transformed(transform) stop = time.time() ims._layer.timePerTile(stop-start, self.tiling.imageRects[tile_no]) self._cache.updateTileIfNecessary( stack_id, ims, tile_no, time.time(), img ) img = self._renderTile( stack_id, tile_no ) self._cache.setTile(stack_id, tile_no, img, self._sims.viewVisible(), self._sims.viewOccluded() ) else: req = (ims, transform, tile_no, stack_id, ims_req, time.time(), self._cache) try: if prefetch: self._prefetchQueue.put_nowait( req ) else: self._dirtyLayerQueue.put_nowait( req ) except Full: msg = " ".join(("Request queue full.", "Dropping tile refresh request.", "Increase queue size!")) warnings.warn(msg) except KeyError: pass def _renderTile( self, stack_id, tile_nr): qimg = None p = None for i, v in enumerate(reversed(self._sims)): visible, layerOpacity, layerImageSource = v if not visible: continue patch = self._cache.layer(stack_id, layerImageSource, tile_nr ) if patch is not None: if qimg is None: qimg = QImage(self.tiling.imageRects[tile_nr].size(), QImage.Format_ARGB32_Premultiplied) qimg.fill(0xffffffff) # Use a hex constant instead. p = QPainter(qimg) p.setOpacity(layerOpacity) p.drawImage(0,0, patch) if p is not None: p.end() return qimg def _onLayerDirty(self, dirtyImgSrc, dataRect ): sceneRect = self.tiling.data2scene.mapRect(dataRect) if dirtyImgSrc in self._sims.viewImageSources(): visibleAndNotOccluded = self._sims.isVisible( dirtyImgSrc ) \ and not self._sims.isOccluded( dirtyImgSrc ) for tile_no in xrange(len(self.tiling)): # an invalid rect means everything is dirty if not sceneRect.isValid() \ or self.tiling.tileRects[tile_no].intersected( sceneRect ): for ims in self._sims.viewImageSources(): self._cache.setLayerDirtyAll(ims, tile_no, True) if visibleAndNotOccluded: self._cache.setTileDirtyAll(tile_no, True) if visibleAndNotOccluded: self.sceneRectChanged.emit( QRectF(sceneRect) ) def _onStackIdChanged( self, oldId, newId ): if newId in self._cache: self._cache.touchStack( newId ) else: self._cache.addStack( newId ) self._current_stack_id = newId self._prefetchQueue = Queue(self._request_queue_size) self.sceneRectChanged.emit(QRectF()) def _onLayerIdChanged( self, ims, oldId, newId ): if self._layerIdChange_means_dirty: self._onLayerDirty( ims, QRect() ) def _onVisibleChanged(self, ims, visible): for tile_no in xrange(len(self.tiling)): self._cache.setTileDirtyAll(tile_no, True) if not self._sims.isOccluded( ims ): self.sceneRectChanged.emit(QRectF()) def _onOpacityChanged(self, ims, opacity): for tile_no in xrange(len(self.tiling)): self._cache.setTileDirtyAll(tile_no, True) if self._sims.isVisible( ims ) and not self._sims.isOccluded( ims ): self.sceneRectChanged.emit(QRectF()) def _onSizeChanged(self): self._cache = _TilesCache(self._current_stack_id, self._sims, maxstacks=self._cache_size) self._dirtyLayerQueue = LifoQueue(self._request_queue_size) self._prefetchQueue = Queue(self._request_queue_size) self.sceneRectChanged.emit(QRectF()) def _onOrderChanged(self): for tile_no in xrange(len(self.tiling)): self._cache.setTileDirtyAll(tile_no, True) self.sceneRectChanged.emit(QRectF())