class ConnectionPool(object):
def __init__(
self, connection_class=Connection,
max_connections=1024, timeout=5, **connection_kwargs):
self.connection_class = connection_class
self.max_connections = max_connections
self.connection_kwargs = connection_kwargs
self.timeout = timeout
self.connections = []
self.reset()
def ensure_safe(self):
if self.pid != os.getpid():
with self.ensure_lock: # lock for concurrent threadings
if self.pid == os.getpid(): # double check
return
self.disconnect()
self.reset()
def get_connection(self):
self.ensure_safe()
try:
connection = self.queue.get(block=True, timeout=self.timeout)
except Empty:
raise ConnectionError('connection pool is full')
if not connection:
connection = self.make_connection()
return connection
def make_connection(self):
connection = self.connection_class(**self.connection_kwargs)
self.connections.append(connection)
return connection
def release(self, connection):
self.ensure_safe()
if connection.pid != self.pid:
return
try:
self.queue.put_nowait(connection)
except Full:
pass
def reset(self):
self.pid = os.getpid()
self.ensure_lock = threading.Lock()
self.disconnect()
# LifoQueue make use of released connections
self.queue = LifoQueue(self.max_connections)
self.connections = []
while True:
try:
self.queue.put_nowait(None)
except Full:
break
def disconnect(self):
for connection in self.connections:
connection.disconnect()
def DFS(N, L, root):
queue = LifoQueue()
queue.put_nowait(root)
while queue.qsize() != 0:
arrangement = queue.get_nowait()
if arrangement.L == L:
return arrangement.Map
else:
search(N, arrangement, queue)
return 0
class TileProvider( QObject ):
THREAD_HEARTBEAT = 0.2
Tile = collections.namedtuple('Tile', 'id qimg rectF progress tiling')
changed = pyqtSignal( QRectF )
'''TileProvider __init__
Keyword Arguments:
cache_size -- maximal number of encountered stacks
to cache, i.e. slices if the imagesources
draw from slicesources (default 10)
request_queue_size -- maximal number of request to queue up (default 100000)
n_threads -- maximal number of request threads; this determines the
maximal number of simultaneously running requests
to the pixelpipeline (default: 2)
layerIdChange_means_dirty -- layerId changes invalidate the cache; by default only
stackId changes do that (default False)
parent -- QObject
'''
def __init__( self, tiling,
stackedImageSources,
cache_size = 10,
request_queue_size = 100000,
n_threads = 2,
layerIdChange_means_dirty=False,
parent=None ):
QObject.__init__( self, parent = parent )
self.tiling = tiling
self._sims = stackedImageSources
self._cache_size = cache_size
self._request_queue_size = request_queue_size
self._n_threads = n_threads
self._layerIdChange_means_dirty = layerIdChange_means_dirty
self._current_stack_id = self._sims.stackId
self._cache = _TilesCache(self._current_stack_id, self._sims, maxstacks=self._cache_size)
self._dirtyLayerQueue = LifoQueue(self._request_queue_size)
self._sims.layerDirty.connect(self._onLayerDirty)
self._sims.visibleChanged.connect(self._onVisibleChanged)
self._sims.opacityChanged.connect(self._onOpacityChanged)
self._sims.sizeChanged.connect(self._onSizeChanged)
self._sims.orderChanged.connect(self._onOrderChanged)
self._sims.stackIdChanged.connect(self._onStackIdChanged)
if self._layerIdChange_means_dirty:
self._sims.layerIdChanged.connect(self._onLayerIdChanged)
self._keepRendering = True
self._dirtyLayerThreads = [Thread(target=self._dirtyLayersWorker) for i in range(self._n_threads)]
for thread in self._dirtyLayerThreads:
thread.daemon = True
[ thread.start() for thread in self._dirtyLayerThreads ]
def getTiles( self, rectF ):
'''Get tiles in rect and request a refresh.
Returns tiles intersectinf with rectF immediatelly and requests a refresh
of these tiles. Next time you call this function the tiles may be already
(partially) updated. If you want to wait until the rendering is fully complete,
call join().
'''
self.requestRefresh( rectF )
tile_nos = self.tiling.intersectedF( rectF )
stack_id = self._current_stack_id
for tile_no in tile_nos:
qimg, progress = self._cache.tile(stack_id, tile_no)
t = TileProvider.Tile(tile_no,
qimg,
QRectF(self.tiling.imageRects[tile_no]),
progress,
self.tiling)
yield t
def requestRefresh( self, rectF ):
'''Requests tiles to be refreshed.
Returns immediatelly. Call join() to wait for
the end of the rendering.
'''
tile_nos = self.tiling.intersectedF( rectF )
for tile_no in tile_nos:
stack_id = self._current_stack_id
self._refreshTile( stack_id, tile_no )
def join( self ):
'''Wait until all refresh request are processed.
Blocks until no refresh request pending anymore and all rendering
finished.
'''
return self._dirtyLayerQueue.join()
def notifyThreadsToStop( self ):
'''Signals render threads to stop.
Call this method at the end of the lifetime of
a TileProvider instance. Otherwise the garbage collector will
not clean up the instance (even if you call del).
'''
self._keepRendering = False
def threadsAreNotifiedToStop( self ):
'''Check if NotifyThreadsToStop() was called at least once.'''
return not self._keepRendering
def joinThreads( self, timeout=None ):
'''Wait until all threads terminated.
Without calling notifyThreadsToStop, threads will never
terminate.
Arguments:
timeout -- timeout in seconds as a floating point number
'''
for thread in self._dirtyLayerThreads:
thread.join( timeout )
def aliveThreads( self ):
'''Return a map of thread identifiers and their alive status.
All threads are alive until notifyThreadsToStop() is
called. After that, they start dying. Call joinThreads() to wait
for the last thread to die.
'''
at = {}
for thread in self._dirtyLayerThreads:
if thread.ident:
at[thread.ident] = thread.isAlive()
return at
def _dirtyLayersWorker( self ):
while self._keepRendering:
try:
ims, tile_nr, stack_id, image_req, timestamp, cache = self._dirtyLayerQueue.get(True, self.THREAD_HEARTBEAT)
except (Empty, TypeError):
#the TypeError occurs when the self._dirtyLayerQueue
#is already None when the thread is being shut down
#on program exit.
#This avoids a lot of warnings.
continue
try:
if timestamp > cache.layerTimestamp( stack_id, ims, tile_nr ):
img = image_req.wait()
cache.updateTileIfNecessary( stack_id, ims, tile_nr, timestamp, img )
if stack_id == self._current_stack_id and cache is self._cache:
self.changed.emit(QRectF(self.tiling.imageRects[tile_nr]))
except KeyError:
pass
finally:
self._dirtyLayerQueue.task_done()
def _refreshTile( self, stack_id, tile_no ):
try:
if self._cache.tileDirty( stack_id, tile_no ):
self._cache.setTileDirty(stack_id, tile_no, False)
img = self._renderTile( stack_id, tile_no )
self._cache.setTile( stack_id, tile_no, img, self._sims.viewVisible(), self._sims.viewOccluded() )
# refresh dirty layer tiles
for ims in self._sims.viewImageSources():
if self._cache.layerDirty(stack_id, ims, tile_no) and not self._sims.isOccluded(ims) and self._sims.isVisible(ims):
req = (ims,
tile_no,
stack_id,
ims.request(self.tiling.imageRects[tile_no]),
time.time(),
self._cache)
try:
self._dirtyLayerQueue.put_nowait( req )
except Full:
warnings.warn("Request queue full. Dropping tile refresh request. Increase queue size!")
except KeyError:
pass
def _renderTile( self, stack_id, tile_nr ):
qimg = QImage(self.tiling.imageRects[tile_nr].size(), QImage.Format_ARGB32_Premultiplied)
qimg.fill(Qt.white)
p = QPainter(qimg)
for i, v in enumerate(reversed(self._sims)):
visible, layerOpacity, layerImageSource = v
if not visible:
continue
patch = self._cache.layer(stack_id, layerImageSource, tile_nr )
if patch is not None:
p.setOpacity(layerOpacity)
p.drawImage(0,0, patch)
p.end()
return qimg
def _onLayerDirty(self, dirtyImgSrc, rect ):
if dirtyImgSrc in self._sims.viewImageSources():
visibleAndNotOccluded = self._sims.isVisible( dirtyImgSrc ) and not self._sims.isOccluded( dirtyImgSrc )
for tile_no in xrange(len(self.tiling)):
#and invalid rect means everything is dirty
if not rect.isValid() or self.tiling.tileRects[tile_no].intersected( rect ):
for ims in self._sims.viewImageSources():
self._cache.setLayerDirtyAll(ims, tile_no, True)
if visibleAndNotOccluded:
self._cache.setTileDirtyAll(tile_no, True)
if visibleAndNotOccluded:
self.changed.emit( QRectF(rect) )
def _onStackIdChanged( self, oldId, newId ):
if newId in self._cache:
self._cache.touchStack( newId )
else:
self._cache.addStack( newId )
self._current_stack_id = newId
self.changed.emit(QRectF())
def _onLayerIdChanged( self, ims, oldId, newId ):
if self._layerIdChange_means_dirty:
self._onLayerDirty( ims, QRect() )
def _onVisibleChanged(self, ims, visible):
for tile_no in xrange(len(self.tiling)):
self._cache.setTileDirtyAll(tile_no, True)
if not self._sims.isOccluded( ims ):
self.changed.emit(QRectF())
def _onOpacityChanged(self, ims, opacity):
for tile_no in xrange(len(self.tiling)):
self._cache.setTileDirtyAll(tile_no, True)
if self._sims.isVisible( ims ) and not self._sims.isOccluded( ims ):
self.changed.emit(QRectF())
def _onSizeChanged(self):
self._cache = _TilesCache(self._current_stack_id, self._sims, maxstacks=self._cache_size)
self._dirtyLayerQueue = LifoQueue(self._request_queue_size)
self.changed.emit(QRectF())
def _onOrderChanged(self):
for tile_no in xrange(len(self.tiling)):
self._cache.setTileDirtyAll(tile_no, True)
self.changed.emit(QRectF())
class ConnectionPool(object):
"""Thread-safe connection pool implementation.
``max_connections``
Maximum number of connections.
``block_timeout``
Maximum number of seconds to wait.
"""
def __init__(self, max_connections=10, block_timeout=5,
**kwds):
self.max_connections = max_connections
self.block_timeout = block_timeout
# kwds is being sent directly to dbapi, so we pop
self._pool_id = kwds.pop('pool_id')
self._logobj = kwds.pop('logobj')
self.kwds = kwds
# make sure max_connections is valid
is_valid = isinstance(max_connections, int) and \
max_connections > 0
if not is_valid:
raise ValueError('max_connections must be a positive int')
# if process id is changed, we will close all connections,
# and reinstantiate this object
self._pid = os.getpid()
# this is where we define the pool, and fill it with None values
self._pool = LifoQueue(max_connections)
while True:
try:
self._pool.put_nowait(None)
except Full:
break
# open connections
self._connections = []
def _checkpid(self):
"""Closes all connections and reinstantiates the object if pid is changed.
"""
if self._pid == os.getpid():
return
self.disconnect()
self.reinstantiate()
def _make_connection(self):
"""Creates a fresh connection.
"""
connection = psycopg2.connect(**self.kwds)
if not hasattr(connection, 'pool_id'):
connection.pool_id = self._pool_id
# we don't need transaction, so let's turn autocommit on
connection.autocommit = True
# pass the logger object if needed
if isinstance(connection, LoggingConnection):
connection.initialize(self._logobj)
_register_hstore(connection, unicode=True)
# encoding
connection.set_client_encoding('UTF8')
# timezone
cursor = connection.cursor()
cursor.execute("set timezone to 'UTC'")
cursor.close()
self._connections.append(connection)
return connection
def get_connection(self):
"""Returns a psycopg2 connection for the given shard_id.
Raises `ConnectionError` if necessary.
"""
self._checkpid()
connection = None
try:
# wait for a connection
connection = self._pool.get(block=True,
timeout=self.block_timeout)
except Empty:
# timeout
raise ConnectionError('no connection available')
# create a new connection if it is not initialized yet
if connection is None:
connection = self._make_connection()
return connection
def put_connection(self, connection):
"""Sends connection back into the pool.
"""
self._checkpid()
try:
self._pool.put_nowait(connection)
except Full:
# reinstantiate may have caused this.
# this connection is useless now.
pass
def disconnect(self):
"""Closes every connection in every pool.
"""
for connection in self._connections:
connection.close()
self._connections = []
def reinstantiate(self):
"""Reinstantiates connection pools.
Make sure you have closed every connection before calling this method.
"""
# let's add these back to kwds
self.kwds['pool_id'] = self._pool_id
self.kwds['logobj'] = self._logobj
self.__init__(max_connections=self.max_connections,
block_timeout=self.block_timeout,
**self.kwds)
class SpeechSynthesizer:
def __init__(self):
# Queue holding the last speech utterance
self._speak_queue = LifoQueue(1)
self._session = Session(profile_name="mylespolly")
self._polly = self._session.client("polly", region_name="eu-west-1")
self._thread = Thread(target=self.run, args=())
self._thread.daemon = True
self._thread.start()
def request(self, text):
"""Clear queue (ignore it being empty) and add text, both non-blocking"""
# for Python 3.4+ this could be written as: with suppress(Empty): ... assuming: from contextlib import suppress
try:
self._speak_queue.get_nowait()
except Empty:
pass
self._speak_queue.put_nowait(text)
def run(self):
"""Continuously process the queue and trigger speech outputs"""
while True:
text = self._speak_queue.get(True, None)
print(text)
try:
response = self._polly.synthesize_speech(Text=text, OutputFormat="mp3", VoiceId="Salli")
except (BotoCoreError, ClientError) as error:
print(error)
sys.exit(-1)
# Access the audio stream from the response
if "AudioStream" in response:
# Note: Closing the stream is important as the service throttles on the
# number of parallel connections. Here we are using contextlib.closing to
# ensure the close method of the stream object will be called automatically
# at the end of the with statement's scope.
with closing(response["AudioStream"]) as stream:
output = os.path.join(gettempdir(), "speech.mp3")
print(output)
try:
# Open a file for writing the output as a binary stream
with open(output, "wb") as file:
file.write(stream.read())
except IOError as error:
# Could not write to file, exit gracefully
print(error)
sys.exit(-1)
else:
# The response didn't contain audio data, exit gracefully
print("Could not stream audio")
sys.exit(-1)
# Play the audio using VLC
# see https://wiki.videolan.org/Python_bindings
# see https://www.olivieraubert.net/vlc/python-ctypes/doc/index.html
p = vlc.MediaPlayer(output)
sleep(0.1)
p.play()
sleep(0.1)
while p.is_playing():
pass
class TileProvider(QObject):
THREAD_HEARTBEAT = 0.2
Tile = collections.namedtuple('Tile', 'id qimg rectF progress tiling')
sceneRectChanged = pyqtSignal(QRectF)
'''TileProvider __init__
Keyword Arguments:
cache_size -- maximal number of encountered stacks
to cache, i.e. slices if the imagesources
draw from slicesources (default 10)
request_queue_size -- maximal number of request to queue up (default 100000)
n_threads -- maximal number of request threads; this determines the
maximal number of simultaneously running requests
to the pixelpipeline (default: 2)
layerIdChange_means_dirty -- layerId changes invalidate the cache; by default only
stackId changes do that (default False)
parent -- QObject
'''
@property
def axesSwapped(self):
return self._axesSwapped
@axesSwapped.setter
def axesSwapped(self, value):
self._axesSwapped = value
def __init__(self,
tiling,
stackedImageSources,
cache_size=100,
request_queue_size=100000,
n_threads=2,
layerIdChange_means_dirty=False,
parent=None):
QObject.__init__(self, parent=parent)
self.tiling = tiling
self.axesSwapped = False
self._sims = stackedImageSources
self._cache_size = cache_size
self._request_queue_size = request_queue_size
self._n_threads = n_threads
self._layerIdChange_means_dirty = layerIdChange_means_dirty
self._current_stack_id = self._sims.stackId
self._cache = _TilesCache(self._current_stack_id,
self._sims,
maxstacks=self._cache_size)
self._dirtyLayerQueue = LifoQueue(self._request_queue_size)
self._prefetchQueue = Queue(self._request_queue_size)
self._sims.layerDirty.connect(self._onLayerDirty)
self._sims.visibleChanged.connect(self._onVisibleChanged)
self._sims.opacityChanged.connect(self._onOpacityChanged)
self._sims.sizeChanged.connect(self._onSizeChanged)
self._sims.orderChanged.connect(self._onOrderChanged)
self._sims.stackIdChanged.connect(self._onStackIdChanged)
if self._layerIdChange_means_dirty:
self._sims.layerIdChanged.connect(self._onLayerIdChanged)
self._keepRendering = True
self._dirtyLayerThreads = [
Thread(target=self._dirtyLayersWorker)
for i in range(self._n_threads)
]
for thread in self._dirtyLayerThreads:
thread.daemon = True
[thread.start() for thread in self._dirtyLayerThreads]
def getTiles(self, rectF):
'''Get tiles in rect and request a refresh.
Returns tiles intersecting with rectF immediately and requests
a refresh of these tiles. Next time you call this function the
tiles may be already (partially) updated. If you want to wait
until the rendering is fully complete, call join().
'''
self.requestRefresh(rectF)
tile_nos = self.tiling.intersected(rectF)
stack_id = self._current_stack_id
for tile_no in tile_nos:
qimg, progress = self._cache.tile(stack_id, tile_no)
yield TileProvider.Tile(tile_no, qimg,
QRectF(self.tiling.imageRects[tile_no]),
progress, self.tiling)
def requestRefresh(self, rectF):
'''Requests tiles to be refreshed.
Returns immediately. Call join() to wait for
the end of the rendering.
'''
tile_nos = self.tiling.intersected(rectF)
for tile_no in tile_nos:
stack_id = self._current_stack_id
self._refreshTile(stack_id, tile_no)
def prefetch(self, rectF, through):
'''Request fetching of tiles in advance.
Returns immediately. Prefetch will commence after all regular
tiles are refreshed (see requestRefresh() and getTiles() ).
The prefetch is reset when the 'through' value of the slicing
changes. Several calls to prefetch are handeled in Fifo
order.
'''
if self._cache_size > 1:
stack_id = (self._current_stack_id[0], enumerate(through))
if stack_id not in self._cache:
self._cache.addStack(stack_id)
self._cache.touchStack(self._current_stack_id)
tile_nos = self.tiling.intersected(rectF)
for tile_no in tile_nos:
self._refreshTile(stack_id, tile_no, prefetch=True)
def join(self):
'''Wait until all refresh request are processed.
Blocks until no refresh request pending anymore and all
rendering finished.
'''
return self._dirtyLayerQueue.join()
def notifyThreadsToStop(self):
'''Signals render threads to stop.
Call this method at the end of the lifetime of a TileProvider
instance. Otherwise the garbage collector will not clean up
the instance (even if you call del).
'''
self._keepRendering = False
def threadsAreNotifiedToStop(self):
'''Check if NotifyThreadsToStop() was called at least once.'''
return not self._keepRendering
def joinThreads(self, timeout=None):
'''Wait until all threads terminated.
Without calling notifyThreadsToStop, threads will never
terminate.
Arguments:
timeout -- timeout in seconds as a floating point number
'''
for thread in self._dirtyLayerThreads:
thread.join(timeout)
def aliveThreads(self):
'''Return a map of thread identifiers and their alive status.
All threads are alive until notifyThreadsToStop() is
called. After that, they start dying. Call joinThreads() to wait
for the last thread to die.
'''
at = {}
for thread in self._dirtyLayerThreads:
if thread.ident:
at[thread.ident] = thread.isAlive()
return at
def _dirtyLayersWorker(self):
while self._keepRendering:
# Save reference to the queue in case self._dirtyLayerQueue reassigned during this pass.
# See onSizeChanged()
dirtyLayerQueue = self._dirtyLayerQueue
prefetchQueue = self._prefetchQueue
try:
try:
result = dirtyLayerQueue.get_nowait()
queue = dirtyLayerQueue
except Empty:
try:
result = prefetchQueue.get_nowait()
queue = prefetchQueue
except Empty:
try:
result = dirtyLayerQueue.get(
True, self.THREAD_HEARTBEAT)
queue = dirtyLayerQueue
except Empty:
continue
except TypeError:
#the TypeError occurs when the queue
#is already None when the thread is being shut down
#on program exit.
#This avoids a lot of warnings.
continue
ims, transform, tile_nr, stack_id, image_req, timestamp, cache = result
try:
try:
layerTimestamp = cache.layerTimestamp(
stack_id, ims, tile_nr)
except KeyError:
pass
else:
if timestamp > layerTimestamp:
img = image_req.wait()
img = img.transformed(transform)
try:
cache.updateTileIfNecessary(
stack_id, ims, tile_nr, timestamp, img)
except KeyError:
pass
else:
if stack_id == self._current_stack_id and cache is self._cache:
self.sceneRectChanged.emit(
QRectF(self.tiling.imageRects[tile_nr]))
except:
with volumina.printLock:
sys.excepthook(*sys.exc_info())
sys.stderr.write(
"ERROR: volumina tiling layer rendering worker thread caught an unhandled exception. See above."
)
finally:
queue.task_done()
def _refreshTile(self, stack_id, tile_no, prefetch=False):
if not self.axesSwapped:
transform = QTransform(0, 1, 0, 1, 0, 0, 1, 1, 1)
else:
transform = QTransform().rotate(90).scale(1, -1)
transform *= self.tiling.data2scene
try:
if self._cache.tileDirty(stack_id, tile_no):
if not prefetch:
self._cache.setTileDirty(stack_id, tile_no, False)
img = self._renderTile(stack_id, tile_no)
self._cache.setTile(stack_id, tile_no, img,
self._sims.viewVisible(),
self._sims.viewOccluded())
# refresh dirty layer tiles
for ims in self._sims.viewImageSources():
if self._cache.layerDirty(stack_id, ims, tile_no) \
and not self._sims.isOccluded(ims) \
and self._sims.isVisible(ims):
rect = self.tiling.imageRects[tile_no]
dataRect = self.tiling.scene2data.mapRect(rect)
ims_req = ims.request(dataRect, stack_id[1])
if ims.direct and not prefetch:
# The ImageSource 'ims' is fast (it has the
# direct flag set to true) so we process
# the request synchronously here. This
# improves the responsiveness for layers
# that have the data readily available.
start = time.time()
img = ims_req.wait()
img = img.transformed(transform)
stop = time.time()
ims._layer.timePerTile(
stop - start, self.tiling.imageRects[tile_no])
self._cache.updateTileIfNecessary(
stack_id, ims, tile_no, time.time(), img)
img = self._renderTile(stack_id, tile_no)
self._cache.setTile(stack_id, tile_no, img,
self._sims.viewVisible(),
self._sims.viewOccluded())
else:
req = (ims, transform, tile_no, stack_id, ims_req,
time.time(), self._cache)
try:
if prefetch:
self._prefetchQueue.put_nowait(req)
else:
self._dirtyLayerQueue.put_nowait(req)
except Full:
msg = " ".join(
("Request queue full.",
"Dropping tile refresh request.",
"Increase queue size!"))
warnings.warn(msg)
except KeyError:
pass
def _renderTile(self, stack_id, tile_nr):
qimg = None
p = None
for i, v in enumerate(reversed(self._sims)):
visible, layerOpacity, layerImageSource = v
if not visible:
continue
patch = self._cache.layer(stack_id, layerImageSource, tile_nr)
if patch is not None:
if qimg is None:
qimg = QImage(self.tiling.imageRects[tile_nr].size(),
QImage.Format_ARGB32_Premultiplied)
qimg.fill(0xffffffff) # Use a hex constant instead.
p = QPainter(qimg)
p.setOpacity(layerOpacity)
p.drawImage(0, 0, patch)
if p is not None:
p.end()
return qimg
def _onLayerDirty(self, dirtyImgSrc, dataRect):
sceneRect = self.tiling.data2scene.mapRect(dataRect)
if dirtyImgSrc in self._sims.viewImageSources():
visibleAndNotOccluded = self._sims.isVisible( dirtyImgSrc ) \
and not self._sims.isOccluded( dirtyImgSrc )
for tile_no in xrange(len(self.tiling)):
# an invalid rect means everything is dirty
if not sceneRect.isValid() \
or self.tiling.tileRects[tile_no].intersected( sceneRect ):
for ims in self._sims.viewImageSources():
self._cache.setLayerDirtyAll(ims, tile_no, True)
if visibleAndNotOccluded:
self._cache.setTileDirtyAll(tile_no, True)
if visibleAndNotOccluded:
self.sceneRectChanged.emit(QRectF(sceneRect))
def _onStackIdChanged(self, oldId, newId):
if newId in self._cache:
self._cache.touchStack(newId)
else:
self._cache.addStack(newId)
self._current_stack_id = newId
self._prefetchQueue = Queue(self._request_queue_size)
self.sceneRectChanged.emit(QRectF())
def _onLayerIdChanged(self, ims, oldId, newId):
if self._layerIdChange_means_dirty:
self._onLayerDirty(ims, QRect())
def _onVisibleChanged(self, ims, visible):
for tile_no in xrange(len(self.tiling)):
self._cache.setTileDirtyAll(tile_no, True)
if not self._sims.isOccluded(ims):
self.sceneRectChanged.emit(QRectF())
def _onOpacityChanged(self, ims, opacity):
for tile_no in xrange(len(self.tiling)):
self._cache.setTileDirtyAll(tile_no, True)
if self._sims.isVisible(ims) and not self._sims.isOccluded(ims):
self.sceneRectChanged.emit(QRectF())
def _onSizeChanged(self):
self._cache = _TilesCache(self._current_stack_id,
self._sims,
maxstacks=self._cache_size)
self._dirtyLayerQueue = LifoQueue(self._request_queue_size)
self._prefetchQueue = Queue(self._request_queue_size)
self.sceneRectChanged.emit(QRectF())
def _onOrderChanged(self):
for tile_no in xrange(len(self.tiling)):
self._cache.setTileDirtyAll(tile_no, True)
self.sceneRectChanged.emit(QRectF())
class TileProvider( QObject ):
THREAD_HEARTBEAT = 0.2
Tile = collections.namedtuple('Tile', 'id qimg rectF progress tiling')
sceneRectChanged = pyqtSignal( QRectF )
'''TileProvider __init__
Keyword Arguments:
cache_size -- maximal number of encountered stacks
to cache, i.e. slices if the imagesources
draw from slicesources (default 10)
request_queue_size -- maximal number of request to queue up (default 100000)
n_threads -- maximal number of request threads; this determines the
maximal number of simultaneously running requests
to the pixelpipeline (default: 2)
layerIdChange_means_dirty -- layerId changes invalidate the cache; by default only
stackId changes do that (default False)
parent -- QObject
'''
@property
def axesSwapped(self):
return self._axesSwapped
@axesSwapped.setter
def axesSwapped(self, value):
self._axesSwapped = value
def __init__( self, tiling, stackedImageSources, cache_size=100,
request_queue_size=100000, n_threads=2,
layerIdChange_means_dirty=False, parent=None ):
QObject.__init__( self, parent = parent )
self.tiling = tiling
self.axesSwapped = False
self._sims = stackedImageSources
self._cache_size = cache_size
self._request_queue_size = request_queue_size
self._n_threads = n_threads
self._layerIdChange_means_dirty = layerIdChange_means_dirty
self._current_stack_id = self._sims.stackId
self._cache = _TilesCache(self._current_stack_id, self._sims,
maxstacks=self._cache_size)
self._dirtyLayerQueue = LifoQueue(self._request_queue_size)
self._prefetchQueue = Queue(self._request_queue_size)
self._sims.layerDirty.connect(self._onLayerDirty)
self._sims.visibleChanged.connect(self._onVisibleChanged)
self._sims.opacityChanged.connect(self._onOpacityChanged)
self._sims.sizeChanged.connect(self._onSizeChanged)
self._sims.orderChanged.connect(self._onOrderChanged)
self._sims.stackIdChanged.connect(self._onStackIdChanged)
if self._layerIdChange_means_dirty:
self._sims.layerIdChanged.connect(self._onLayerIdChanged)
self._keepRendering = True
self._dirtyLayerThreads = [Thread(target=self._dirtyLayersWorker)
for i in range(self._n_threads)]
for thread in self._dirtyLayerThreads:
thread.daemon = True
[ thread.start() for thread in self._dirtyLayerThreads ]
def getTiles( self, rectF ):
'''Get tiles in rect and request a refresh.
Returns tiles intersecting with rectF immediately and requests
a refresh of these tiles. Next time you call this function the
tiles may be already (partially) updated. If you want to wait
until the rendering is fully complete, call join().
'''
self.requestRefresh( rectF )
tile_nos = self.tiling.intersected( rectF )
stack_id = self._current_stack_id
for tile_no in tile_nos:
qimg, progress = self._cache.tile(stack_id, tile_no)
yield TileProvider.Tile(
tile_no,
qimg,
QRectF(self.tiling.imageRects[tile_no]),
progress,
self.tiling)
def requestRefresh( self, rectF ):
'''Requests tiles to be refreshed.
Returns immediately. Call join() to wait for
the end of the rendering.
'''
tile_nos = self.tiling.intersected( rectF )
for tile_no in tile_nos:
stack_id = self._current_stack_id
self._refreshTile( stack_id, tile_no )
def prefetch( self, rectF, through ):
'''Request fetching of tiles in advance.
Returns immediately. Prefetch will commence after all regular
tiles are refreshed (see requestRefresh() and getTiles() ).
The prefetch is reset when the 'through' value of the slicing
changes. Several calls to prefetch are handeled in Fifo
order.
'''
if self._cache_size > 1:
stack_id = (self._current_stack_id[0], through)
if stack_id not in self._cache:
self._cache.addStack(stack_id)
self._cache.touchStack( self._current_stack_id )
tile_nos = self.tiling.intersected( rectF )
for tile_no in tile_nos:
self._refreshTile( stack_id, tile_no, prefetch=True )
def join( self ):
'''Wait until all refresh request are processed.
Blocks until no refresh request pending anymore and all
rendering finished.
'''
return self._dirtyLayerQueue.join()
def notifyThreadsToStop( self ):
'''Signals render threads to stop.
Call this method at the end of the lifetime of a TileProvider
instance. Otherwise the garbage collector will not clean up
the instance (even if you call del).
'''
self._keepRendering = False
def threadsAreNotifiedToStop( self ):
'''Check if NotifyThreadsToStop() was called at least once.'''
return not self._keepRendering
def joinThreads( self, timeout=None ):
'''Wait until all threads terminated.
Without calling notifyThreadsToStop, threads will never
terminate.
Arguments:
timeout -- timeout in seconds as a floating point number
'''
for thread in self._dirtyLayerThreads:
thread.join( timeout )
def aliveThreads( self ):
'''Return a map of thread identifiers and their alive status.
All threads are alive until notifyThreadsToStop() is
called. After that, they start dying. Call joinThreads() to wait
for the last thread to die.
'''
at = {}
for thread in self._dirtyLayerThreads:
if thread.ident:
at[thread.ident] = thread.isAlive()
return at
def _dirtyLayersWorker( self ):
while self._keepRendering:
# Save reference to the queue in case self._dirtyLayerQueue reassigned during this pass.
# See onSizeChanged()
dirtyLayerQueue = self._dirtyLayerQueue
prefetchQueue = self._prefetchQueue
try:
try:
result = dirtyLayerQueue.get_nowait()
queue = dirtyLayerQueue
except Empty:
try:
result = prefetchQueue.get_nowait()
queue = prefetchQueue
except Empty:
try:
result = dirtyLayerQueue.get(True, self.THREAD_HEARTBEAT)
queue = dirtyLayerQueue
except Empty:
continue
except TypeError:
#the TypeError occurs when the queue
#is already None when the thread is being shut down
#on program exit.
#This avoids a lot of warnings.
continue
ims, transform, tile_nr, stack_id, image_req, timestamp, cache = result
try:
try:
layerTimestamp = cache.layerTimestamp( stack_id, ims, tile_nr )
except KeyError:
pass
else:
if timestamp > layerTimestamp:
img = image_req.wait()
img = img.transformed(transform)
try:
cache.updateTileIfNecessary( stack_id, ims, tile_nr, timestamp, img )
except KeyError:
pass
else:
if stack_id == self._current_stack_id and cache is self._cache:
self.sceneRectChanged.emit(QRectF(self.tiling.imageRects[tile_nr]))
except:
with volumina.printLock:
sys.excepthook( *sys.exc_info() )
sys.stderr.write("ERROR: volumina tiling layer rendering worker thread caught an unhandled exception. See above.")
finally:
queue.task_done()
def _refreshTile( self, stack_id, tile_no, prefetch=False ):
if not self.axesSwapped:
transform = QTransform(0,1,0,1,0,0,1,1,1)
else:
transform = QTransform().rotate(90).scale(1,-1)
transform *= self.tiling.data2scene
try:
if self._cache.tileDirty( stack_id, tile_no ):
if not prefetch:
self._cache.setTileDirty(stack_id, tile_no, False)
img = self._renderTile( stack_id, tile_no )
self._cache.setTile(stack_id, tile_no, img,
self._sims.viewVisible(),
self._sims.viewOccluded())
# refresh dirty layer tiles
for ims in self._sims.viewImageSources():
if self._cache.layerDirty(stack_id, ims, tile_no) \
and not self._sims.isOccluded(ims) \
and self._sims.isVisible(ims):
rect = self.tiling.imageRects[tile_no]
dataRect = self.tiling.scene2data.mapRect(rect)
ims_req = ims.request(dataRect, stack_id[1])
if ims.direct:
# The ImageSource 'ims' is fast (it has the
# direct flag set to true) so we process
# the request synchronously here. This
# improves the responsiveness for layers
# that have the data readily available.
start = time.time()
img = ims_req.wait()
img = img.transformed(transform)
stop = time.time()
ims._layer.timePerTile(stop-start,
self.tiling.imageRects[tile_no])
self._cache.updateTileIfNecessary(
stack_id, ims, tile_no, time.time(), img )
img = self._renderTile( stack_id, tile_no )
self._cache.setTile(stack_id, tile_no,
img, self._sims.viewVisible(),
self._sims.viewOccluded() )
else:
req = (ims, transform, tile_no, stack_id,
ims_req, time.time(), self._cache)
try:
if prefetch:
self._prefetchQueue.put_nowait( req )
else:
self._dirtyLayerQueue.put_nowait( req )
except Full:
msg = " ".join(("Request queue full.",
"Dropping tile refresh request.",
"Increase queue size!"))
warnings.warn(msg)
except KeyError:
pass
def _renderTile( self, stack_id, tile_nr):
qimg = None
p = None
for i, v in enumerate(reversed(self._sims)):
visible, layerOpacity, layerImageSource = v
if not visible:
continue
patch = self._cache.layer(stack_id, layerImageSource, tile_nr )
if patch is not None:
if qimg is None:
qimg = QImage(self.tiling.imageRects[tile_nr].size(), QImage.Format_ARGB32_Premultiplied)
qimg.fill(0xffffffff) # Use a hex constant instead.
p = QPainter(qimg)
p.setOpacity(layerOpacity)
p.drawImage(0,0, patch)
if p is not None:
p.end()
return qimg
def _onLayerDirty(self, dirtyImgSrc, dataRect ):
sceneRect = self.tiling.data2scene.mapRect(dataRect)
if dirtyImgSrc in self._sims.viewImageSources():
visibleAndNotOccluded = self._sims.isVisible( dirtyImgSrc ) \
and not self._sims.isOccluded( dirtyImgSrc )
for tile_no in xrange(len(self.tiling)):
# an invalid rect means everything is dirty
if not sceneRect.isValid() \
or self.tiling.tileRects[tile_no].intersected( sceneRect ):
for ims in self._sims.viewImageSources():
self._cache.setLayerDirtyAll(ims, tile_no, True)
if visibleAndNotOccluded:
self._cache.setTileDirtyAll(tile_no, True)
if visibleAndNotOccluded:
self.sceneRectChanged.emit( QRectF(sceneRect) )
def _onStackIdChanged( self, oldId, newId ):
if newId in self._cache:
self._cache.touchStack( newId )
else:
self._cache.addStack( newId )
self._current_stack_id = newId
self._prefetchQueue = Queue(self._request_queue_size)
self.sceneRectChanged.emit(QRectF())
def _onLayerIdChanged( self, ims, oldId, newId ):
if self._layerIdChange_means_dirty:
self._onLayerDirty( ims, QRect() )
def _onVisibleChanged(self, ims, visible):
for tile_no in xrange(len(self.tiling)):
self._cache.setTileDirtyAll(tile_no, True)
if not self._sims.isOccluded( ims ):
self.sceneRectChanged.emit(QRectF())
def _onOpacityChanged(self, ims, opacity):
for tile_no in xrange(len(self.tiling)):
self._cache.setTileDirtyAll(tile_no, True)
if self._sims.isVisible( ims ) and not self._sims.isOccluded( ims ):
self.sceneRectChanged.emit(QRectF())
def _onSizeChanged(self):
self._cache = _TilesCache(self._current_stack_id, self._sims,
maxstacks=self._cache_size)
self._dirtyLayerQueue = LifoQueue(self._request_queue_size)
self._prefetchQueue = Queue(self._request_queue_size)
self.sceneRectChanged.emit(QRectF())
def _onOrderChanged(self):
for tile_no in xrange(len(self.tiling)):
self._cache.setTileDirtyAll(tile_no, True)
self.sceneRectChanged.emit(QRectF())
