class SaveManager(QObject):
start_save = pyqtSignal()
report_error = pyqtSignal(object)
save_done = pyqtSignal()
def __init__(self, parent, notify=None):
QObject.__init__(self, parent)
self.count = 0
self.last_saved = -1
self.requests = LifoQueue()
self.notify_requests = LifoQueue()
self.notify_data = notify
t = Thread(name='save-thread', target=self.run)
t.daemon = True
t.start()
t = Thread(name='notify-thread', target=self.notify_calibre)
t.daemon = True
t.start()
self.status_widget = w = SaveWidget(parent)
self.start_save.connect(w.start, type=Qt.QueuedConnection)
self.save_done.connect(w.stop, type=Qt.QueuedConnection)
def schedule(self, tdir, container):
self.count += 1
self.requests.put((self.count, tdir, container))
def run(self):
while True:
x = self.requests.get()
if x is None:
self.requests.task_done()
self.__empty_queue()
break
try:
count, tdir, container = x
self.process_save(count, tdir, container)
except:
import traceback
traceback.print_exc()
finally:
self.requests.task_done()
def notify_calibre(self):
while True:
if not self.notify_requests.get():
break
send_message(self.notify_data)
def clear_notify_data(self):
self.notify_data = None
def __empty_queue(self):
' Only to be used during shutdown '
while True:
try:
self.requests.get_nowait()
except Empty:
break
else:
self.requests.task_done()
def process_save(self, count, tdir, container):
if count <= self.last_saved:
shutil.rmtree(tdir, ignore_errors=True)
return
self.last_saved = count
self.start_save.emit()
try:
self.do_save(tdir, container)
except:
import traceback
self.report_error.emit(traceback.format_exc())
self.save_done.emit()
if self.notify_data:
self.notify_requests.put(True)
def do_save(self, tdir, container):
try:
save_container(container, container.path_to_ebook)
finally:
shutil.rmtree(tdir, ignore_errors=True)
@property
def has_tasks(self):
return bool(self.requests.unfinished_tasks)
def wait(self, timeout=30):
if timeout is None:
self.requests.join()
else:
try:
join_with_timeout(self.requests, timeout)
except RuntimeError:
return False
return True
def shutdown(self):
self.requests.put(None)
self.notify_requests.put(None)
class SaveManager(QObject):
start_save = pyqtSignal()
report_error = pyqtSignal(object)
save_done = pyqtSignal()
def __init__(self, parent):
QObject.__init__(self, parent)
self.count = 0
self.last_saved = -1
self.requests = LifoQueue()
t = Thread(name='save-thread', target=self.run)
t.daemon = True
t.start()
self.status_widget = w = SaveWidget(parent)
self.start_save.connect(w.start, type=Qt.QueuedConnection)
self.save_done.connect(w.stop, type=Qt.QueuedConnection)
def schedule(self, tdir, container):
self.count += 1
self.requests.put((self.count, tdir, container))
def run(self):
while True:
x = self.requests.get()
if x is None:
self.requests.task_done()
self.__empty_queue()
break
try:
count, tdir, container = x
self.process_save(count, tdir, container)
except:
import traceback
traceback.print_exc()
finally:
self.requests.task_done()
def __empty_queue(self):
' Only to be used during shutdown '
while True:
try:
self.requests.get_nowait()
except Empty:
break
else:
self.requests.task_done()
def process_save(self, count, tdir, container):
if count <= self.last_saved:
shutil.rmtree(tdir, ignore_errors=True)
return
self.last_saved = count
self.start_save.emit()
try:
self.do_save(tdir, container)
except:
import traceback
self.report_error.emit(traceback.format_exc())
self.save_done.emit()
def do_save(self, tdir, container):
try:
save_container(container, container.path_to_ebook)
finally:
shutil.rmtree(tdir, ignore_errors=True)
@property
def has_tasks(self):
return bool(self.requests.unfinished_tasks)
def wait(self, timeout=30):
if timeout is None:
self.requests.join()
else:
try:
join_with_timeout(self.requests, timeout)
except RuntimeError:
return False
return True
def shutdown(self):
self.requests.put(None)
class TileProvider( QObject ):
THREAD_HEARTBEAT = 0.2
Tile = collections.namedtuple('Tile', 'id qimg rectF progress tiling')
changed = pyqtSignal( QRectF )
'''TileProvider __init__
Keyword Arguments:
cache_size -- maximal number of encountered stacks
to cache, i.e. slices if the imagesources
draw from slicesources (default 10)
request_queue_size -- maximal number of request to queue up (default 100000)
n_threads -- maximal number of request threads; this determines the
maximal number of simultaneously running requests
to the pixelpipeline (default: 2)
layerIdChange_means_dirty -- layerId changes invalidate the cache; by default only
stackId changes do that (default False)
parent -- QObject
'''
def __init__( self, tiling,
stackedImageSources,
cache_size = 10,
request_queue_size = 100000,
n_threads = 2,
layerIdChange_means_dirty=False,
parent=None ):
QObject.__init__( self, parent = parent )
self.tiling = tiling
self._sims = stackedImageSources
self._cache_size = cache_size
self._request_queue_size = request_queue_size
self._n_threads = n_threads
self._layerIdChange_means_dirty = layerIdChange_means_dirty
self._current_stack_id = self._sims.stackId
self._cache = _TilesCache(self._current_stack_id, self._sims, maxstacks=self._cache_size)
self._dirtyLayerQueue = LifoQueue(self._request_queue_size)
self._sims.layerDirty.connect(self._onLayerDirty)
self._sims.visibleChanged.connect(self._onVisibleChanged)
self._sims.opacityChanged.connect(self._onOpacityChanged)
self._sims.sizeChanged.connect(self._onSizeChanged)
self._sims.orderChanged.connect(self._onOrderChanged)
self._sims.stackIdChanged.connect(self._onStackIdChanged)
if self._layerIdChange_means_dirty:
self._sims.layerIdChanged.connect(self._onLayerIdChanged)
self._keepRendering = True
self._dirtyLayerThreads = [Thread(target=self._dirtyLayersWorker) for i in range(self._n_threads)]
for thread in self._dirtyLayerThreads:
thread.daemon = True
[ thread.start() for thread in self._dirtyLayerThreads ]
def getTiles( self, rectF ):
'''Get tiles in rect and request a refresh.
Returns tiles intersectinf with rectF immediatelly and requests a refresh
of these tiles. Next time you call this function the tiles may be already
(partially) updated. If you want to wait until the rendering is fully complete,
call join().
'''
self.requestRefresh( rectF )
tile_nos = self.tiling.intersectedF( rectF )
stack_id = self._current_stack_id
for tile_no in tile_nos:
qimg, progress = self._cache.tile(stack_id, tile_no)
t = TileProvider.Tile(tile_no,
qimg,
QRectF(self.tiling.imageRects[tile_no]),
progress,
self.tiling)
yield t
def requestRefresh( self, rectF ):
'''Requests tiles to be refreshed.
Returns immediatelly. Call join() to wait for
the end of the rendering.
'''
tile_nos = self.tiling.intersectedF( rectF )
for tile_no in tile_nos:
stack_id = self._current_stack_id
self._refreshTile( stack_id, tile_no )
def join( self ):
'''Wait until all refresh request are processed.
Blocks until no refresh request pending anymore and all rendering
finished.
'''
return self._dirtyLayerQueue.join()
def notifyThreadsToStop( self ):
'''Signals render threads to stop.
Call this method at the end of the lifetime of
a TileProvider instance. Otherwise the garbage collector will
not clean up the instance (even if you call del).
'''
self._keepRendering = False
def threadsAreNotifiedToStop( self ):
'''Check if NotifyThreadsToStop() was called at least once.'''
return not self._keepRendering
def joinThreads( self, timeout=None ):
'''Wait until all threads terminated.
Without calling notifyThreadsToStop, threads will never
terminate.
Arguments:
timeout -- timeout in seconds as a floating point number
'''
for thread in self._dirtyLayerThreads:
thread.join( timeout )
def aliveThreads( self ):
'''Return a map of thread identifiers and their alive status.
All threads are alive until notifyThreadsToStop() is
called. After that, they start dying. Call joinThreads() to wait
for the last thread to die.
'''
at = {}
for thread in self._dirtyLayerThreads:
if thread.ident:
at[thread.ident] = thread.isAlive()
return at
def _dirtyLayersWorker( self ):
while self._keepRendering:
try:
ims, tile_nr, stack_id, image_req, timestamp, cache = self._dirtyLayerQueue.get(True, self.THREAD_HEARTBEAT)
except (Empty, TypeError):
#the TypeError occurs when the self._dirtyLayerQueue
#is already None when the thread is being shut down
#on program exit.
#This avoids a lot of warnings.
continue
try:
if timestamp > cache.layerTimestamp( stack_id, ims, tile_nr ):
img = image_req.wait()
cache.updateTileIfNecessary( stack_id, ims, tile_nr, timestamp, img )
if stack_id == self._current_stack_id and cache is self._cache:
self.changed.emit(QRectF(self.tiling.imageRects[tile_nr]))
except KeyError:
pass
finally:
self._dirtyLayerQueue.task_done()
def _refreshTile( self, stack_id, tile_no ):
try:
if self._cache.tileDirty( stack_id, tile_no ):
self._cache.setTileDirty(stack_id, tile_no, False)
img = self._renderTile( stack_id, tile_no )
self._cache.setTile( stack_id, tile_no, img, self._sims.viewVisible(), self._sims.viewOccluded() )
# refresh dirty layer tiles
for ims in self._sims.viewImageSources():
if self._cache.layerDirty(stack_id, ims, tile_no) and not self._sims.isOccluded(ims) and self._sims.isVisible(ims):
req = (ims,
tile_no,
stack_id,
ims.request(self.tiling.imageRects[tile_no]),
time.time(),
self._cache)
try:
self._dirtyLayerQueue.put_nowait( req )
except Full:
warnings.warn("Request queue full. Dropping tile refresh request. Increase queue size!")
except KeyError:
pass
def _renderTile( self, stack_id, tile_nr ):
qimg = QImage(self.tiling.imageRects[tile_nr].size(), QImage.Format_ARGB32_Premultiplied)
qimg.fill(Qt.white)
p = QPainter(qimg)
for i, v in enumerate(reversed(self._sims)):
visible, layerOpacity, layerImageSource = v
if not visible:
continue
patch = self._cache.layer(stack_id, layerImageSource, tile_nr )
if patch is not None:
p.setOpacity(layerOpacity)
p.drawImage(0,0, patch)
p.end()
return qimg
def _onLayerDirty(self, dirtyImgSrc, rect ):
if dirtyImgSrc in self._sims.viewImageSources():
visibleAndNotOccluded = self._sims.isVisible( dirtyImgSrc ) and not self._sims.isOccluded( dirtyImgSrc )
for tile_no in xrange(len(self.tiling)):
#and invalid rect means everything is dirty
if not rect.isValid() or self.tiling.tileRects[tile_no].intersected( rect ):
for ims in self._sims.viewImageSources():
self._cache.setLayerDirtyAll(ims, tile_no, True)
if visibleAndNotOccluded:
self._cache.setTileDirtyAll(tile_no, True)
if visibleAndNotOccluded:
self.changed.emit( QRectF(rect) )
def _onStackIdChanged( self, oldId, newId ):
if newId in self._cache:
self._cache.touchStack( newId )
else:
self._cache.addStack( newId )
self._current_stack_id = newId
self.changed.emit(QRectF())
def _onLayerIdChanged( self, ims, oldId, newId ):
if self._layerIdChange_means_dirty:
self._onLayerDirty( ims, QRect() )
def _onVisibleChanged(self, ims, visible):
for tile_no in xrange(len(self.tiling)):
self._cache.setTileDirtyAll(tile_no, True)
if not self._sims.isOccluded( ims ):
self.changed.emit(QRectF())
def _onOpacityChanged(self, ims, opacity):
for tile_no in xrange(len(self.tiling)):
self._cache.setTileDirtyAll(tile_no, True)
if self._sims.isVisible( ims ) and not self._sims.isOccluded( ims ):
self.changed.emit(QRectF())
def _onSizeChanged(self):
self._cache = _TilesCache(self._current_stack_id, self._sims, maxstacks=self._cache_size)
self._dirtyLayerQueue = LifoQueue(self._request_queue_size)
self.changed.emit(QRectF())
def _onOrderChanged(self):
for tile_no in xrange(len(self.tiling)):
self._cache.setTileDirtyAll(tile_no, True)
self.changed.emit(QRectF())
class B2BucketThreadedLocal(B2Bucket):
def __init__(self, *args):
super(B2BucketThreaded, self).__init__(*args)
num_threads = 50
self.queue = LifoQueue(num_threads * 2)
self.file_locks = defaultdict(Lock)
self.running = True
self.threads = []
print "Thread ",
for i in xrange(num_threads):
t = threading.Thread(target=self._file_updater)
t.start()
self.threads.append(t)
print ".",
print
self.pre_queue_lock = Lock()
self.pre_queue_running = True
self.pre_queue = LifoQueue(num_threads * 2)
self.pre_file_dict = {}
self.pre_thread = threading.Thread(target=self._prepare_update)
self.pre_thread.start()
def _prepare_update(self):
while self.pre_queue_running:
try:
filename, local_filename, operation = self.pre_queue.get(
True, 1)
self.pre_file_dict[filename] = (time(), local_filename,
operation)
self.pre_queue.task_done()
except Empty:
for filename, (timestamp, local_filename,
operation) in self.pre_file_dict.items():
if time() - timestamp > 15:
self.queue.put((filename, local_filename, operation))
del self.pre_file_dict[filename]
for filename, (timestamp, local_filename,
operation) in self.pre_file_dict.items():
self.queue.put((filename, local_filename, operation))
del self.pre_file_dict[filename]
def _file_updater(self):
while self.running:
try:
filename, local_filename, operation = self.queue.get(True, 1)
except Empty:
continue
with self.file_locks[filename]:
if operation == "deletion":
super(B2BucketThreaded, self)._delete_file(filename)
self.queue.task_done()
elif operation == "upload":
super(B2BucketThreaded,
self)._put_file(filename, local_filename)
self.queue.task_done()
elif operation == "download":
super(B2BucketThreaded,
self)._get_file(filename, local_filename)
self.queue.task_done()
else:
self.logger.error("Invalid operation %s on %s" %
(operation, filename))
def __enter__(self):
return self
def __exit__(self, *args, **kwargs):
self.logger.info("Waiting for all B2 requests to complete")
self.logger.info("Pre-Queue contains %s elements",
self.pre_queue.qsize())
self.pre_queue.join()
self.logger.info("Joining pre queue thread")
self.pre_queue_running = False
self.pre_thread.join()
self.logger.info("Queue contains %s elements", self.queue.qsize())
self.queue.join()
self.logger.info("Joining threads")
self.running = False
for t in self.threads:
t.join()
def put_file(self, filename, local_filename):
with self.pre_queue_lock:
self.logger.info("Postponing upload of %s (%s)", filename,
len(data))
self.pre_queue.put((filename, local_filename, "upload"), True)
new_file = {}
new_file['fileName'] = filename
new_file['fileId'] = None
new_file['uploadTimestamp'] = time()
new_file['action'] = 'upload'
new_file['contentLength'] = len(data)
return new_file
def delete_file(self, filename):
with self.pre_queue_lock:
self.logger.info("Postponing deletion of %s", filename)
self.pre_queue.put((filename, None, "deletion"), True)
def get_file(self, filename, local_filename):
with self.pre_queue_lock:
self.logger.info("Postponing download of %s", filename)
self.pre_queue.put((filename, local_filename, "download"), True)
class Imagery:
# Number of simultaneous connections to imagery provider and concurrent placement layouts.
# Limit to 1 to prevent worker threads slowing UI too much.
connections = 1
def __init__(self, canvas):
self.providers = {
'Bing': self.bing_setup,
'ArcGIS': self.arcgis_setup,
'MapQuest': self.mq_setup
}
self.canvas = canvas
self.imageryprovider = None
self.provider_base = None
self.provider_url = None
self.provider_logo = None # (filename, width, height)
self.provider_levelmin = self.provider_levelmax = 0
self.placementcache = {
} # previously created placements (or None if image couldn't be loaded), indexed by quadkey.
# placement may not be laid out if image is still being fetched.
self.tile = (0, 999) # X-Plane 1x1degree tile - [lat,lon] of SW
self.loc = None
self.dist = 0
self.filecache = Filecache()
# Setup a pool of worker threads
self.workers = []
self.q = LifoQueue()
for i in range(Imagery.connections):
t = threading.Thread(target=self.worker)
t.daemon = True # this doesn't appear to work for threads blocked on Queue
t.start()
self.workers.append(t)
# Worker thread
def worker(self):
# Each thread gets its own tessellators in thread-local storage
tls = threading.local()
tls.tess = gluNewTess()
gluTessNormal(tls.tess, 0, -1, 0)
gluTessProperty(tls.tess, GLU_TESS_WINDING_RULE,
GLU_TESS_WINDING_NEGATIVE)
gluTessCallback(tls.tess, GLU_TESS_VERTEX_DATA,
ElevationMeshBase.tessvertex)
gluTessCallback(tls.tess, GLU_TESS_EDGE_FLAG,
ElevationMeshBase.tessedge)
tls.csgt = gluNewTess()
gluTessNormal(tls.csgt, 0, -1, 0)
gluTessProperty(tls.csgt, GLU_TESS_WINDING_RULE,
GLU_TESS_WINDING_ABS_GEQ_TWO)
gluTessCallback(tls.csgt, GLU_TESS_VERTEX_DATA,
ElevationMeshBase.tessvertex)
gluTessCallback(tls.csgt, GLU_TESS_COMBINE,
ElevationMeshBase.tesscombinetris)
gluTessCallback(tls.csgt, GLU_TESS_EDGE_FLAG,
ElevationMeshBase.tessedge)
while True:
(fn, args) = self.q.get()
if not fn: exit() # Die!
fn(tls, *args)
self.q.task_done()
def exit(self):
# Closing down
self.filecache.writedir()
# kill workers
for i in range(Imagery.connections):
self.q.put(
(None, ())) # Top priority! Don't want to hold up program exit
# wait for them
for t in self.workers:
t.join()
def reset(self):
# Called on reload or on new tile. Empty the cache and forget allocations since:
# a) in the case of reload, textures have been dropped and so would need to be reloaded anyway;
# b) try to limit cluttering up the VBO with allocations we may not need again;
# c) images by straddle multiple tiles and these would need to be recalculated anyway.
for placement in self.placementcache.itervalues():
if placement: placement.clearlayout()
self.placementcache = {}
def goto(self, imageryprovider, loc, dist, screensize):
if not imageryprovider: imageryprovider = None
if imageryprovider != self.imageryprovider:
self.provider_base = None
self.provider_url = None
self.provider_logo = None
self.imageryprovider = imageryprovider
if self.imageryprovider not in self.providers: return
self.q.put((self.providers[self.imageryprovider], ()))
newtile = (int(floor(loc[0])), int(floor(loc[1])))
if not self.provider_url or self.tile != newtile:
# New tile - drop cache of Clutter
for placement in self.placementcache.itervalues():
if placement: placement.clearlayout()
self.placementcache = {}
self.tile = newtile
self.loc = loc
self.placements(dist, screensize) # Kick off any image loading
# Return placements to be drawn. May allocate into vertexcache as a side effect.
def placements(self, dist, screensize):
level0mpp = 2 * pi * 6378137 / 256 # metres per pixel at level 0
if screensize.width <= 0 or not (int(self.loc[0]) or int(
self.loc[1])) or not self.provider_url:
return [
] # Don't do anything on startup. Can't do anything without a valid provider.
# layout assumes mesh loaded
assert (
int(floor(self.loc[0])), int(floor(self.loc[1])),
prefs.options & Prefs.ELEVATION
) in self.canvas.vertexcache.elevation, '%s %s' % (
(int(floor(self.loc[0])), int(floor(self.loc[1])), prefs.options
& Prefs.ELEVATION), self.canvas.vertexcache.elevation.keys())
# http://msdn.microsoft.com/en-us/library/bb259689.aspx
# http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Resolution_and_Scale
width = dist + dist # Width in m of screen (from glOrtho setup)
ppm = screensize.width / width # Pixels on screen required by 1 metre, ignoring tilt
level = min(
int(round(log(ppm * level0mpp * cos(radians(self.loc[0])), 2))),
self.provider_levelmax) # zoom level required
levelmin = max(
13, self.provider_levelmin
) # arbitrary - tessellating out at higher levels takes too long
level = max(level, levelmin + 1)
ntiles = 2**level # number of tiles per axis at this level
#mpp=cos(radians(self.loc[0]))*level0mpp/ntiles # actual resolution at this level
#coverage=width/(256*mpp) # how many tiles to cover screen width - in practice varies between ~2.4 and ~4.8
(cx, cy) = self.latlon2xy(self.loc[0], self.loc[1],
level) # centre tile
#print self.loc, width, screensize.width, 1/ppm, ppm, level, ntiles, cx, cy
if __debug__: print "Desire imagery level", level
# We're using a Lifo queue, so as the user navigates the most important tiles are processed first.
# Should remove from the queue those tiles the user is probably not going to see again, but that's difficult so we don't.
# Display 6x6 tiles if available that cover the same area as 3x3 at the next higher level (this is to prevent weirdness when zooming in)
cx = 2 * (cx / 2)
cy = 2 * (cy / 2)
placements = []
needed = set(
) # Placements at this level failed either cos imagery not available at this location/level or is pending layout
fetch = []
seq = [(-2, 3),
(-1, 3), (0, 3), (1, 3), (2, 3), (3, 3), (3, 2), (3, 1), (3, 0),
(3, -1), (3, -2), (2, -2), (1, -2), (0, -2), (-1, -2), (-2, -2),
(-2, -1), (-2, 0), (-2, 1), (-2, 2), (-1, 2), (0, 2), (1, 2),
(2, 2), (2, 1), (2, 0), (2, -1), (1, -1), (0, -1), (-1, -1),
(-1, 0), (-1, 1), (0, 1), (1, 1), (1, 0), (0, 0)]
for i in range(len(seq)):
(x, y) = seq[i]
placement = self.getplacement(cx + x, cy + y, level,
False) # Don't initiate fetch yet
if placement and placement.islaidout():
placements.append(placement)
else:
needed.add(((cx + x) / 2, (cy + y) / 2))
if 0 <= x <= 1 and 0 <= y <= 1:
fetch.append(
(cx + x, cy + y, level,
True)) # schedule fetch of the centre 2x2 tiles
# Go up and get 5x5 tiles around the centre tile - but only draw them if higher-res imagery not (yet) available.
level -= 1
cx /= 2
cy /= 2
fail2 = True
if self.q.empty():
# If the queue is empty then the first (and low importance) tile starts processing immediately.
# So here we add the most important centre tile of 5x5 and ensure it starts processing.
placement = self.getplacement(cx, cy, level,
True) # Initiate fetch
if placement:
fail2 = False # Some imagery may be available at this level
if placement.islaidout() and (cx, cy) in needed:
placements.insert(
0, placement
) # Insert at start so drawn under higher-level
needed.remove((cx, cy))
while not self.q.empty():
time.sleep(0) # Allow worker thread to remove from queue
# First initiate fetch of higher-level imagery of centre 2x2
for args in fetch:
self.getplacement(*args)
seq = [(1, -2), (0, -2), (-1, -2), (-2, -1), (-2, 0), (-2, 1), (-1, 2),
(0, 2), (1, 2), (2, 1), (2, 0), (2, -1), (1, -1), (0, -1),
(-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1), (1, 0), (0, 0)
] # 5x5 with corners removed #, (0,3), (3,0), (0,-3), (-3,0)]
for i in range(len(seq)):
(x, y) = seq[i]
placement = self.getplacement(cx + x, cy + y, level,
True) # Initiate fetch
if placement:
fail2 = False # Some imagery may be available at this level
if placement.islaidout() and (abs(x) > 1 or abs(y) > 1 or
(cx + x, cy + y) in needed):
placements.insert(
0, placement
) # Insert at start so drawn under higher-level
while fail2 and level > levelmin:
# No imagery available at all at higher level. Go up and get 3x3 tiles around the centre tile.
level -= 1
cx /= 2
cy /= 2
seq = [(1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1),
(1, 1), (1, 0), (0, 0)]
for i in range(len(seq)):
(x, y) = seq[i]
placement = self.getplacement(cx + x, cy + y, level, True)
if placement:
fail2 = False # Some imagery may be available at this level
if placement.islaidout():
placements.insert(
0, placement
) # Insert at start so drawn under higher-level
if __debug__: print "Actual imagery level", level
return placements
# Helper to return coordinates in a spiral from http://stackoverflow.com/questions/398299/looping-in-a-spiral
def spiral(self, N, M):
x = y = 0
dx, dy = 0, -1
retval = []
for dumb in xrange(N * M):
if abs(x) == abs(y) and [dx, dy] != [1, 0] or x > 0 and y == 1 - x:
dx, dy = -dy, dx # corner, change direction
if abs(x) > N / 2 or abs(y) > M / 2: # non-square
dx, dy = -dy, dx # change direction
x, y = -y + dx, x + dy # jump
retval.append((x, y))
x, y = x + dx, y + dy
return retval
# Returns a laid-out placement if possible, or not laid-out if image is still loading, or None if not available.
def getplacement(self, x, y, level, fetch):
(name, url) = self.provider_url(x, y, level)
if name in self.placementcache:
# Already created
placement = self.placementcache[name]
elif fetch:
# Make a new one. We could also do this if the image file is available, but don't since layout is expensive.
(north, west) = self.xy2latlon(x, y, level)
(south, east) = self.xy2latlon(x + 1, y + 1, level)
placement = DrapedImage(name, 65535, [[
Node([west, north, 0, 1]),
Node([east, north, 1, 1]),
Node([east, south, 1, 0]),
Node([west, south, 0, 0])
]])
placement.load(self.canvas.lookup, self.canvas.defs,
self.canvas.vertexcache)
self.placementcache[name] = placement
# Initiate fetch of image and do layout. Prioritise more detail.
self.q.put((self.initplacement, (placement, name, url)))
else:
placement = None
# Load it if it's not loaded but is ready to be
if placement and not placement.islaidout() and Polygon.islaidout(
placement):
try:
if __debug__: clock = time.clock()
filename = self.filecache.get(name) # downloaded image or None
self.canvas.vertexcache.allocate_dynamic(
placement, True) # couldn't do this in thread context
placement.definition.texture = self.canvas.vertexcache.texcache.get(
filename, wrap=False, downsample=False, fixsize=True)
if __debug__:
print "%6.3f time in imagery load for %s" % (
time.clock() - clock, placement.name)
assert placement.islaidout()
except:
if __debug__: print_exc()
# Some failure - perhaps corrupted image?
placement.clearlayout()
placement = None
self.placementcache[name] = None
return placement
def latlon2xy(self, lat, lon, level):
ntiles = 2**level # number of tiles per axis at this level
sinlat = sin(radians(lat))
x = int((lon + 180) * ntiles / 360.0)
y = int((0.5 - (log((1 + sinlat) / (1 - sinlat)) / fourpi)) * ntiles)
return (x, y)
def xy2latlon(self, x, y, level):
ntiles = float(2**level) # number of tiles per axis at this level
lat = 90 - 360 * atan(exp((y / ntiles - 0.5) * 2 * pi)) / pi
lon = x * 360.0 / ntiles - 180
return (lat, lon)
def bing_quadkey(self, x, y, level):
# http://msdn.microsoft.com/en-us/library/bb259689.aspx
i = level
quadkey = ''
while i > 0:
digit = 0
mask = 1 << (i - 1)
if (x & mask):
digit += 1
if (y & mask):
digit += 2
quadkey += ('%d' % digit)
i -= 1
url = self.provider_base % quadkey
name = basename(url).split('?')[0]
return (name, url)
# Called in worker thread - don't do anything fancy since main body of code is not thread-safe
def bing_setup(self, tls):
try:
key = 'AhATjCXv4Sb-i_YKsa_8lF4DtHwVoicFxl0Stc9QiXZNywFbI2rajKZCsLFIMOX2'
h = urlopen(
'http://dev.virtualearth.net/REST/v1/Imagery/Metadata/Aerial?key=%s'
% key)
d = h.read()
h.close()
info = json_decode(d)
# http://msdn.microsoft.com/en-us/library/ff701707.aspx
if 'authenticationResultCode' not in info or info[
'authenticationResultCode'] != 'ValidCredentials' or 'statusCode' not in info or info[
'statusCode'] != 200:
return
res = info['resourceSets'][0]['resources'][0]
# http://msdn.microsoft.com/en-us/library/ff701712.aspx
self.provider_levelmin = int(res['zoomMin'])
self.provider_levelmax = int(res['zoomMax'])
self.provider_base = res['imageUrl'].replace(
'{subdomain}', res['imageUrlSubdomains'][-1]
).replace('{culture}', 'en').replace(
'{quadkey}', '%s'
) + '&key=' + key # was random.choice(res['imageUrlSubdomains']) but always picking the same server seems to give better caching
self.provider_url = self.bing_quadkey
if info['brandLogoUri']:
filename = self.filecache.fetch(basename(info['brandLogoUri']),
info['brandLogoUri'])
if filename:
image = PIL.Image.open(
filename) # yuck. but at least open is lazy
self.provider_logo = (filename, image.size[0],
image.size[1])
except:
if __debug__: print_exc()
self.canvas.Refresh() # Might have been waiting on this to get imagery
def arcgis_url(self, x, y, level):
url = self.provider_base % ("%d/%d/%d" % (level, y, x))
name = "arcgis_%d_%d_%d.jpeg" % (level, y, x)
return (name, url)
# Called in worker thread - don't do anything fancy since main body of code is not thread-safe
def arcgis_setup(self, tls):
try:
h = urlopen(
'http://services.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer?f=json'
)
d = h.read()
h.close()
info = json_decode(d)
# http://resources.arcgis.com/en/help/rest/apiref/index.html
# http://resources.arcgis.com/en/help/rest/apiref/mapserver.html
self.provider_levelmin = min(
[lod['level'] for lod in info['tileInfo']['lods']])
self.provider_levelmax = max(
[lod['level'] for lod in info['tileInfo']['lods']])
self.provider_base = 'http://services.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/%s'
self.provider_url = self.arcgis_url
filename = self.filecache.fetch(
'logo-med.png',
'http://serverapi.arcgisonline.com/jsapi/arcgis/2.8/images/map/logo-med.png'
)
if filename:
image = PIL.Image.open(
filename) # yuck. but at least open is lazy
self.provider_logo = (filename, image.size[0], image.size[1])
except:
if __debug__: print_exc()
self.canvas.Refresh() # Might have been waiting on this to get imagery
def mq_url(self, x, y, level):
url = self.provider_base % ("%d/%d/%d.jpg" % (level, x, y))
name = "mq_%d_%d_%d.jpg" % (level, x, y)
return (name, url)
# Called in worker thread - don't do anything fancy since main body of code is not thread-safe
def mq_setup(self, tls):
# http://developer.mapquest.com/web/products/open/map
try:
self.provider_levelmin = 0
self.provider_levelmax = 18
self.provider_base = 'http://otile1.mqcdn.com/tiles/1.0.0/map/%s'
self.provider_url = self.mq_url
filename = self.filecache.fetch(
'questy.png',
'http://open.mapquest.com/cdn/toolkit/lite/images/questy.png')
if filename:
image = PIL.Image.open(
filename) # yuck. but at least open is lazy
self.provider_logo = (filename, image.size[0], image.size[1])
except:
if __debug__: print_exc()
self.canvas.Refresh() # Might have been waiting on this to get imagery
# Called in worker thread - fetches image and does placement layout (which uses it's own tessellator and so is thread-safe).
# don't do anything fancy since main body of code is not thread-safe
def initplacement(self, tls, placement, name, url):
filename = self.filecache.fetch(name, url)
if not filename:
# Couldn't fetch image - remove corresponding placement
self.placementcache[name] = None
else:
if __debug__: clock = time.clock()
placement.layout(self.tile, tls=tls)
if not placement.dynamic_data.size:
if __debug__:
print "DrapedImage layout failed for %s - no tris" % placement.name
self.placementcache[name] = None
elif __debug__:
print "%6.3f time in imagery layout for %s" % (
time.clock() - clock, placement.name)
self.canvas.Refresh(
) # Probably wanting to display this - corresponding placement will be loaded and laid out during OnPaint
class ThreadPool(object):
def __init__(self, threadNum, max_tasks_per_period=10, seconds_per_period=30):
self.pool = [] # 线程池
self.threadNum = threadNum # 线程数
self.runningLock = Lock() # 线程锁
self.taskLock = Lock() # getTask函数的锁
self.running = 0 # 正在run的线程数
# 设置为LIFO队列：在抓取了第一个post的页面后，随后需要添加所有其后的评论页，
# 使用LIFO队列可以保证尽快将第一个post的所有评论抓取到，并存储
self.taskQueue = LifoQueue() # 任务队列
# 一分钟内允许的最大访问次数
self.max_tasks_per_period = max_tasks_per_period
# 定制每分钟含有的秒数
self.seconds_per_period = seconds_per_period
# 当前周期内已经访问的网页数量
self.currentPeriodVisits = 0
# 将一分钟当作一个访问周期，记录当前周期的开始时间
self.periodStart = time.time() # 使用当前时间初始化
def startThreads(self):
"""Create a certain number of threads and started to run
All Workers share the same ThreadPool
"""
# 开始当前的抓取周期
self.periodStart = time.time()
for i in range(self.threadNum):
self.pool.append(Worker(self, i))
def stopThreads(self):
for thread in self.pool:
thread.stop()
thread.join()
del self.pool[:]
def putTask(self, func, *args, **kargs):
self.taskQueue.put((func, args, kargs))
def getTask(self, *args, **kargs):
# 进行访问控制: 判断当前周期内访问的网页数目是否大于最大数目
if self.currentPeriodVisits >= self.max_tasks_per_period - 2:
timeNow = time.time()
seconds = timeNow - self.periodStart
if seconds < self.seconds_per_period: # 如果当前还没有过一分钟,则sleep
remain = self.seconds_per_period - seconds
print "ThreadPool Waiting for " + str(remain) + " seconds."
time.sleep(int(remain + 1))
self.periodStart = time.time() # 重新设置开始时间
self.currentPeriodVisits = 0
try:
# task = self.taskQueue.get(*args, **kargs)
task = self.taskQueue.get_nowait()
except Empty:
return (None, None, None)
self.currentPeriodVisits += 1
return task
def taskJoin(self, *args, **kargs):
"""Queue.join: Blocks until all items in the queue have been gotten and processed.
"""
self.taskQueue.join()
def taskDone(self, *args, **kargs):
self.taskQueue.task_done()
def increaseRunsNum(self):
self.runningLock.acquire()
self.running += 1 # 正在运行的线程数加1
self.runningLock.release()
def decreaseRunsNum(self):
self.runningLock.acquire()
self.running -= 1
self.runningLock.release()
def getTaskLeft(self):
# 线程池的所有任务包括：
# taskQueue中未被下载的任务, resultQueue中完成了但是还没被取出的任务, 正在运行的任务
# 因此任务总数为三者之和
return self.taskQueue.qsize() + self.running
class Imagery:
# Number of simultaneous connections to imagery provider and concurrent placement layouts.
# Limit to 1 to prevent worker threads slowing UI too much.
connections=1
def __init__(self, canvas):
self.providers={'Bing': self.bing_setup, 'ArcGIS': self.arcgis_setup, 'MapQuest': self.mq_setup }
self.canvas=canvas
self.imageryprovider=None
self.provider_base=None
self.provider_url=None
self.provider_logo=None # (filename, width, height)
self.provider_levelmin=self.provider_levelmax=0
self.placementcache={} # previously created placements (or None if image couldn't be loaded), indexed by quadkey.
# placement may not be laid out if image is still being fetched.
self.tile=(0,999) # X-Plane 1x1degree tile - [lat,lon] of SW
self.loc=None
self.dist=0
self.filecache=Filecache()
# Setup a pool of worker threads
self.workers=[]
self.q=LifoQueue()
for i in range(Imagery.connections):
t=threading.Thread(target=self.worker)
t.daemon=True # this doesn't appear to work for threads blocked on Queue
t.start()
self.workers.append(t)
# Worker thread
def worker(self):
# Each thread gets its own tessellators in thread-local storage
tls=threading.local()
tls.tess=gluNewTess()
gluTessNormal(tls.tess, 0, -1, 0)
gluTessProperty(tls.tess, GLU_TESS_WINDING_RULE, GLU_TESS_WINDING_NEGATIVE)
gluTessCallback(tls.tess, GLU_TESS_VERTEX_DATA, ElevationMeshBase.tessvertex)
gluTessCallback(tls.tess, GLU_TESS_EDGE_FLAG, ElevationMeshBase.tessedge)
tls.csgt=gluNewTess()
gluTessNormal(tls.csgt, 0, -1, 0)
gluTessProperty(tls.csgt, GLU_TESS_WINDING_RULE, GLU_TESS_WINDING_ABS_GEQ_TWO)
gluTessCallback(tls.csgt, GLU_TESS_VERTEX_DATA, ElevationMeshBase.tessvertex)
gluTessCallback(tls.csgt, GLU_TESS_COMBINE, ElevationMeshBase.tesscombinetris)
gluTessCallback(tls.csgt, GLU_TESS_EDGE_FLAG, ElevationMeshBase.tessedge)
while True:
(fn, args)=self.q.get()
if not fn: exit() # Die!
fn(tls, *args)
self.q.task_done()
def exit(self):
# Closing down
self.filecache.writedir()
# kill workers
for i in range(Imagery.connections):
self.q.put((None, ())) # Top priority! Don't want to hold up program exit
# wait for them
for t in self.workers:
t.join()
def reset(self):
# Called on reload or on new tile. Empty the cache and forget allocations since:
# a) in the case of reload, textures have been dropped and so would need to be reloaded anyway;
# b) try to limit cluttering up the VBO with allocations we may not need again;
# c) images by straddle multiple tiles and these would need to be recalculated anyway.
for placement in self.placementcache.itervalues():
if placement: placement.clearlayout()
self.placementcache={}
def goto(self, imageryprovider, loc, dist, screensize):
if not imageryprovider: imageryprovider=None
if imageryprovider!=self.imageryprovider:
self.provider_base=None
self.provider_url=None
self.provider_logo=None
self.imageryprovider=imageryprovider
if self.imageryprovider not in self.providers: return
self.q.put((self.providers[self.imageryprovider], ()))
newtile=(int(floor(loc[0])),int(floor(loc[1])))
if not self.provider_url or self.tile!=newtile:
# New tile - drop cache of Clutter
for placement in self.placementcache.itervalues():
if placement: placement.clearlayout()
self.placementcache={}
self.tile=newtile
self.loc=loc
self.placements(dist, screensize) # Kick off any image loading
# Return placements to be drawn. May allocate into vertexcache as a side effect.
def placements(self, dist, screensize):
level0mpp=2*pi*6378137/256 # metres per pixel at level 0
if screensize.width<=0 or not (int(self.loc[0]) or int(self.loc[1])) or not self.provider_url:
return [] # Don't do anything on startup. Can't do anything without a valid provider.
# layout assumes mesh loaded
assert (int(floor(self.loc[0])),int(floor(self.loc[1])),prefs.options&Prefs.ELEVATION) in self.canvas.vertexcache.elevation, '%s %s' % ((int(floor(self.loc[0])),int(floor(self.loc[1])),prefs.options&Prefs.ELEVATION), self.canvas.vertexcache.elevation.keys())
# http://msdn.microsoft.com/en-us/library/bb259689.aspx
# http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Resolution_and_Scale
width=dist+dist # Width in m of screen (from glOrtho setup)
ppm=screensize.width/width # Pixels on screen required by 1 metre, ignoring tilt
level=min(int(round(log(ppm*level0mpp*cos(radians(self.loc[0])), 2))), self.provider_levelmax) # zoom level required
levelmin=max(13, self.provider_levelmin) # arbitrary - tessellating out at higher levels takes too long
level=max(level,levelmin+1)
ntiles=2**level # number of tiles per axis at this level
#mpp=cos(radians(self.loc[0]))*level0mpp/ntiles # actual resolution at this level
#coverage=width/(256*mpp) # how many tiles to cover screen width - in practice varies between ~2.4 and ~4.8
(cx,cy)=self.latlon2xy(self.loc[0], self.loc[1], level) # centre tile
#print self.loc, width, screensize.width, 1/ppm, ppm, level, ntiles, cx, cy
if __debug__: print "Desire imagery level", level
# We're using a Lifo queue, so as the user navigates the most important tiles are processed first.
# Should remove from the queue those tiles the user is probably not going to see again, but that's difficult so we don't.
# Display 6x6 tiles if available that cover the same area as 3x3 at the next higher level (this is to prevent weirdness when zooming in)
cx=2*(cx/2)
cy=2*(cy/2)
placements=[]
needed=set() # Placements at this level failed either cos imagery not available at this location/level or is pending layout
fetch=[]
seq=[(-2, 3), (-1, 3), (0, 3), (1, 3), (2, 3), (3, 3), (3, 2), (3, 1), (3, 0), (3, -1), (3, -2), (2, -2), (1, -2), (0, -2), (-1, -2), (-2, -2), (-2, -1), (-2, 0), (-2, 1), (-2, 2), (-1, 2), (0, 2), (1, 2), (2, 2), (2, 1), (2, 0), (2, -1), (1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1), (1, 0), (0, 0)]
for i in range(len(seq)):
(x,y)=seq[i]
placement=self.getplacement(cx+x,cy+y,level, False) # Don't initiate fetch yet
if placement and placement.islaidout():
placements.append(placement)
else:
needed.add(((cx+x)/2,(cy+y)/2))
if 0<=x<=1 and 0<=y<=1:
fetch.append((cx+x,cy+y,level, True)) # schedule fetch of the centre 2x2 tiles
# Go up and get 5x5 tiles around the centre tile - but only draw them if higher-res imagery not (yet) available.
level-=1
cx/=2
cy/=2
fail2=True
if self.q.empty():
# If the queue is empty then the first (and low importance) tile starts processing immediately.
# So here we add the most important centre tile of 5x5 and ensure it starts processing.
placement=self.getplacement(cx,cy,level, True) # Initiate fetch
if placement:
fail2=False # Some imagery may be available at this level
if placement.islaidout() and (cx,cy) in needed:
placements.insert(0,placement) # Insert at start so drawn under higher-level
needed.remove((cx,cy))
while not self.q.empty():
time.sleep(0) # Allow worker thread to remove from queue
# First initiate fetch of higher-level imagery of centre 2x2
for args in fetch: self.getplacement(*args)
seq=[(1, -2), (0, -2), (-1, -2), (-2, -1), (-2, 0), (-2, 1), (-1, 2), (0, 2), (1, 2), (2, 1), (2, 0), (2, -1), (1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1), (1, 0), (0, 0)] # 5x5 with corners removed #, (0,3), (3,0), (0,-3), (-3,0)]
for i in range(len(seq)):
(x,y)=seq[i]
placement=self.getplacement(cx+x,cy+y,level, True) # Initiate fetch
if placement:
fail2=False # Some imagery may be available at this level
if placement.islaidout() and (abs(x)>1 or abs(y)>1 or (cx+x,cy+y) in needed):
placements.insert(0,placement) # Insert at start so drawn under higher-level
while fail2 and level>levelmin:
# No imagery available at all at higher level. Go up and get 3x3 tiles around the centre tile.
level-=1
cx/=2
cy/=2
seq=[(1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1), (1, 0), (0, 0)]
for i in range(len(seq)):
(x,y)=seq[i]
placement=self.getplacement(cx+x,cy+y,level, True)
if placement:
fail2=False # Some imagery may be available at this level
if placement.islaidout():
placements.insert(0,placement) # Insert at start so drawn under higher-level
if __debug__: print "Actual imagery level", level
return placements
# Helper to return coordinates in a spiral from http://stackoverflow.com/questions/398299/looping-in-a-spiral
def spiral(self, N, M):
x = y = 0
dx, dy = 0, -1
retval=[]
for dumb in xrange(N*M):
if abs(x) == abs(y) and [dx,dy] != [1,0] or x>0 and y == 1-x:
dx, dy = -dy, dx # corner, change direction
if abs(x)>N/2 or abs(y)>M/2: # non-square
dx, dy = -dy, dx # change direction
x, y = -y+dx, x+dy # jump
retval.append((x,y))
x, y = x+dx, y+dy
return retval
# Returns a laid-out placement if possible, or not laid-out if image is still loading, or None if not available.
def getplacement(self,x,y,level,fetch):
(name,url)=self.provider_url(x,y,level)
if name in self.placementcache:
# Already created
placement=self.placementcache[name]
elif fetch:
# Make a new one. We could also do this if the image file is available, but don't since layout is expensive.
(north,west)=self.xy2latlon(x,y,level)
(south,east)=self.xy2latlon(x+1,y+1,level)
placement=DrapedImage(name, 65535, [[Node([west,north,0,1]),Node([east,north,1,1]),Node([east,south,1,0]),Node([west,south,0,0])]])
placement.load(self.canvas.lookup, self.canvas.defs, self.canvas.vertexcache)
self.placementcache[name]=placement
# Initiate fetch of image and do layout. Prioritise more detail.
self.q.put((self.initplacement, (placement,name,url)))
else:
placement=None
# Load it if it's not loaded but is ready to be
if placement and not placement.islaidout() and Polygon.islaidout(placement):
try:
if __debug__: clock=time.clock()
filename=self.filecache.get(name) # downloaded image or None
self.canvas.vertexcache.allocate_dynamic(placement, True) # couldn't do this in thread context
placement.definition.texture=self.canvas.vertexcache.texcache.get(filename, wrap=False, downsample=False, fixsize=True)
if __debug__: print "%6.3f time in imagery load for %s" % (time.clock()-clock, placement.name)
assert placement.islaidout()
except:
if __debug__: print_exc()
# Some failure - perhaps corrupted image?
placement.clearlayout()
placement=None
self.placementcache[name]=None
return placement
def latlon2xy(self, lat, lon, level):
ntiles=2**level # number of tiles per axis at this level
sinlat=sin(radians(lat))
x = int((lon+180) * ntiles/360.0)
y = int( (0.5 - (log((1+sinlat)/(1-sinlat)) / fourpi)) * ntiles)
return (x,y)
def xy2latlon(self, x, y, level):
ntiles=float(2**level) # number of tiles per axis at this level
lat = 90 - 360 * atan(exp((y/ntiles-0.5)*2*pi)) / pi
lon = x*360.0/ntiles - 180
return (lat,lon)
def bing_quadkey(self, x, y, level):
# http://msdn.microsoft.com/en-us/library/bb259689.aspx
i=level
quadkey=''
while i>0:
digit=0
mask = 1 << (i-1)
if (x & mask):
digit+=1
if (y & mask):
digit+=2
quadkey+=('%d' % digit)
i-=1
url=self.provider_base % quadkey
name=basename(url).split('?')[0]
return (name,url)
# Called in worker thread - don't do anything fancy since main body of code is not thread-safe
def bing_setup(self, tls):
try:
key='AhATjCXv4Sb-i_YKsa_8lF4DtHwVoicFxl0Stc9QiXZNywFbI2rajKZCsLFIMOX2'
h=urlopen('http://dev.virtualearth.net/REST/v1/Imagery/Metadata/Aerial?key=%s' % key)
d=h.read()
h.close()
info=json_decode(d)
# http://msdn.microsoft.com/en-us/library/ff701707.aspx
if 'authenticationResultCode' not in info or info['authenticationResultCode']!='ValidCredentials' or 'statusCode' not in info or info['statusCode']!=200:
return
res=info['resourceSets'][0]['resources'][0]
# http://msdn.microsoft.com/en-us/library/ff701712.aspx
self.provider_levelmin=int(res['zoomMin'])
self.provider_levelmax=int(res['zoomMax'])
self.provider_base=res['imageUrl'].replace('{subdomain}',res['imageUrlSubdomains'][-1]).replace('{culture}','en').replace('{quadkey}','%s') + '&key=' + key # was random.choice(res['imageUrlSubdomains']) but always picking the same server seems to give better caching
self.provider_url=self.bing_quadkey
if info['brandLogoUri']:
filename=self.filecache.fetch(basename(info['brandLogoUri']), info['brandLogoUri'])
if filename:
image = PIL.Image.open(filename) # yuck. but at least open is lazy
self.provider_logo=(filename,image.size[0],image.size[1])
except:
if __debug__: print_exc()
self.canvas.Refresh() # Might have been waiting on this to get imagery
def arcgis_url(self, x, y, level):
url=self.provider_base % ("%d/%d/%d" % (level, y, x))
name="arcgis_%d_%d_%d.jpeg" % (level, y, x)
return (name,url)
# Called in worker thread - don't do anything fancy since main body of code is not thread-safe
def arcgis_setup(self, tls):
try:
h=urlopen('http://services.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer?f=json')
d=h.read()
h.close()
info=json_decode(d)
# http://resources.arcgis.com/en/help/rest/apiref/index.html
# http://resources.arcgis.com/en/help/rest/apiref/mapserver.html
self.provider_levelmin=min([lod['level'] for lod in info['tileInfo']['lods']])
self.provider_levelmax=max([lod['level'] for lod in info['tileInfo']['lods']])
self.provider_base='http://services.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/%s'
self.provider_url=self.arcgis_url
filename=self.filecache.fetch('logo-med.png', 'http://serverapi.arcgisonline.com/jsapi/arcgis/2.8/images/map/logo-med.png')
if filename:
image = PIL.Image.open(filename) # yuck. but at least open is lazy
self.provider_logo=(filename,image.size[0],image.size[1])
except:
if __debug__: print_exc()
self.canvas.Refresh() # Might have been waiting on this to get imagery
def mq_url(self, x, y, level):
url=self.provider_base % ("%d/%d/%d.jpg" % (level, x, y))
name="mq_%d_%d_%d.jpg" % (level, x, y)
return (name,url)
# Called in worker thread - don't do anything fancy since main body of code is not thread-safe
def mq_setup(self, tls):
# http://developer.mapquest.com/web/products/open/map
try:
self.provider_levelmin=0
self.provider_levelmax=18
self.provider_base='http://otile1.mqcdn.com/tiles/1.0.0/map/%s'
self.provider_url=self.mq_url
filename=self.filecache.fetch('questy.png', 'http://open.mapquest.com/cdn/toolkit/lite/images/questy.png')
if filename:
image = PIL.Image.open(filename) # yuck. but at least open is lazy
self.provider_logo=(filename,image.size[0],image.size[1])
except:
if __debug__: print_exc()
self.canvas.Refresh() # Might have been waiting on this to get imagery
# Called in worker thread - fetches image and does placement layout (which uses it's own tessellator and so is thread-safe).
# don't do anything fancy since main body of code is not thread-safe
def initplacement(self, tls, placement, name, url):
filename=self.filecache.fetch(name, url)
if not filename:
# Couldn't fetch image - remove corresponding placement
self.placementcache[name]=None
else:
if __debug__: clock=time.clock()
placement.layout(self.tile, tls=tls)
if not placement.dynamic_data.size:
if __debug__: print "DrapedImage layout failed for %s - no tris" % placement.name
self.placementcache[name]=None
elif __debug__:
print "%6.3f time in imagery layout for %s" % (time.clock()-clock, placement.name)
self.canvas.Refresh() # Probably wanting to display this - corresponding placement will be loaded and laid out during OnPaint
class Transceiver(object):
def __init__(self, config={}, message_handler=None):
"""Set up a receiver which connects to the messaging hub.
:param config: This is a dict in the form::
config = dict(
incoming='tcp://localhost:15566', # default
outgoing='tcp://localhost:15567',
idle_timeout=1000, # milliseconds:
)
"""
self.log = logging.getLogger("evasion.messenger.endpoint.Transceiver")
self.endpoint_uuid = str(uuid.uuid4())
self.exit_time = threading.Event()
self.wait_for_exit = threading.Event()
self.incoming = None # configured in main().
self.incoming_uri = config.get("incoming", 'tcp://localhost:15566')
self.log.info("Recieving on <%s>" % self.incoming_uri)
self.outgoing_uri = config.get("outgoing", 'tcp://localhost:15567')
self.log.info("Sending on <%s>" % self.outgoing_uri)
self.idle_timeout = int(config.get("idle_timeout", 2000))
self.log.info("Idle Timeout (ms): %d" % self.idle_timeout)
self.message_handler = message_handler
self.sync_message = frames.sync_message(
"endpoint-%s" % self.endpoint_uuid
)
# Queue up messages to be sent in the main message loop
self._out_queue = LifoQueue()
def main(self):
"""Running the main loop sending and receiving.
This will keep running until stop() is called. This
sets the exit flag causing clean up and shutdown.
"""
self.exitTime = False
context = zmq.Context()
incoming = context.socket(zmq.SUB)
incoming.setsockopt(zmq.SUBSCRIBE, '')
incoming.connect(self.incoming_uri)
outgoing = context.socket(zmq.PUSH);
outgoing.connect(self.outgoing_uri);
def _shutdown():
try:
incoming.close()
except ZMQError:
self.log.exception("main: error calling incoming.close()")
try:
outgoing.close()
except ZMQError:
self.log.exception("main: error calling outgoing.close()")
try:
context.term()
except ZMQError:
self.log.exception("main: error calling context.term()")
try:
poller = zmq.Poller()
poller.register(incoming, zmq.POLLIN)
while not self.exit_time.is_set():
try:
events = poller.poll(self.idle_timeout)
except ZMQError as e:
# 4 = 'Interrupted system call'
if e.errno == 4:
self.log.info("main: exit time: %s" % e)
break
else:
self.log.info("main: <%s>" % e)
break
except Exception:
self.log.exception("main: fatal error while polling ")
break
else:
if (events > 0):
msg = incoming.recv_multipart()
self.message_in(tuple(msg))
# Now recover and queued outgoing messages:
if not self._out_queue.empty():
message = self._out_queue.get_nowait()
if message:
try:
# send sync hub followed by message. The sync
# will kick the hub into life if its just
# started:
outgoing.send_multipart(self.sync_message)
outgoing.send_multipart(message)
except ZMQError as e:
# 4 = 'Interrupted system call'
if e.errno == 4:
self.log.info((
"main: sigint or other signal interrupt"
", exit time <%s>"
) % e)
break
else:
self.log.info("main: <%s>" % e)
break
except Exception:
self.log.exception("main: fatal error sending ")
break
finally:
self._out_queue.task_done()
finally:
self.wait_for_exit.set()
_shutdown()
def start(self):
"""Set up zmq communication and start receiving messages from the hub.
"""
# coverage can't seem to get to this:
def _main(notused): # pragma: no cover
self.exit_time.clear()
self.wait_for_exit.clear()
self.main()
thread.start_new(_main, (0,))
def stop(self, wait=2):
"""Stop receiving messages from the hub and clean up.
:param wait: The time in seconds to wait before giving up
on a clean shutdown.
"""
self.log.info("stop: shutting down messaging.")
self.exit_time.set()
self.wait_for_exit.wait(wait)
self.log.info("stop: done.")
def message_out(self, message):
"""This sends a message to the messagehub for dispatch to all connected
endpoints.
:param message: A tuple or list representing a multipart ZMQ message.
If the message is not a tuple or list then MessageOutError
will be raised.
Note: The message is actually queued here so that the main loop will
send it when its ready.
:returns: None.
"""
if isinstance(message, list) or isinstance(message, tuple):
self._out_queue.put(message)
else:
m = "The message must be a list or tuple instead of <%s>" % type(
message
)
raise MessageOutError(m)
def message_in(self, message):
"""Called on receipt of an evasion frame to determine what to do.
The message_handler set in the constructer will be called if one
was set. If none was set then the message will be logged at the
DEBUG level.
:param message: A tuple or list representing a multipart ZMQ message.
:returns: None.
"""
if self.message_handler:
try:
#self.log.debug("message_in: message <%s>" % str(message))
self.message_handler(message)
except:
self.log.exception("message_in: Error handling message - ")
else:
self.log.debug("message_in: message <%s>" % str(message))
class B2BucketThreadedLocal(B2Bucket):
def __init__(self, *args):
super(B2BucketThreaded, self).__init__( *args)
num_threads=50
self.queue = LifoQueue(num_threads*2)
self.file_locks = defaultdict(Lock)
self.running = True
self.threads = []
print "Thread ",
for i in xrange(num_threads):
t = threading.Thread(target=self._file_updater)
t.start()
self.threads.append(t)
print ".",
print
self.pre_queue_lock = Lock()
self.pre_queue_running = True
self.pre_queue = LifoQueue(num_threads*2)
self.pre_file_dict = {}
self.pre_thread = threading.Thread(target=self._prepare_update)
self.pre_thread.start()
def _prepare_update(self):
while self.pre_queue_running:
try:
filename, local_filename, operation = self.pre_queue.get(True,1)
self.pre_file_dict[filename] = (time(), local_filename, operation)
self.pre_queue.task_done()
except Empty:
for filename, (timestamp, local_filename, operation) in self.pre_file_dict.items():
if time()-timestamp > 15:
self.queue.put((filename, local_filename, operation))
del self.pre_file_dict[filename]
for filename, (timestamp, local_filename, operation) in self.pre_file_dict.items():
self.queue.put((filename, local_filename, operation))
del self.pre_file_dict[filename]
def _file_updater(self):
while self.running:
try:
filename, local_filename, operation = self.queue.get(True,1)
except Empty:
continue
with self.file_locks[filename]:
if operation == "deletion":
super(B2BucketThreaded,self)._delete_file(filename)
self.queue.task_done()
elif operation == "upload":
super(B2BucketThreaded,self)._put_file(filename, local_filename)
self.queue.task_done()
elif operation == "download":
super(B2BucketThreaded,self)._get_file(filename, local_filename)
self.queue.task_done()
else:
self.logger.error("Invalid operation %s on %s" % (operation, filename))
def __enter__(self):
return self
def __exit__(self, *args, **kwargs):
self.logger.info("Waiting for all B2 requests to complete")
self.logger.info("Pre-Queue contains %s elements", self.pre_queue.qsize())
self.pre_queue.join()
self.logger.info("Joining pre queue thread")
self.pre_queue_running = False
self.pre_thread.join()
self.logger.info("Queue contains %s elements", self.queue.qsize())
self.queue.join()
self.logger.info("Joining threads")
self.running = False
for t in self.threads:
t.join()
def put_file(self, filename, local_filename):
with self.pre_queue_lock:
self.logger.info("Postponing upload of %s (%s)", filename, len(data))
self.pre_queue.put((filename, local_filename, "upload"), True)
new_file = {}
new_file['fileName'] = filename
new_file['fileId'] = None
new_file['uploadTimestamp'] = time()
new_file['action'] = 'upload'
new_file['contentLength'] = len(data)
return new_file
def delete_file(self, filename):
with self.pre_queue_lock:
self.logger.info("Postponing deletion of %s", filename)
self.pre_queue.put((filename, None, "deletion"),True)
def get_file(self, filename, local_filename):
with self.pre_queue_lock:
self.logger.info("Postponing download of %s", filename)
self.pre_queue.put((filename, local_filename, "download"),True)
class SaveManager(QObject):
start_save = pyqtSignal()
report_error = pyqtSignal(object)
save_done = pyqtSignal()
def __init__(self, parent):
QObject.__init__(self, parent)
self.count = 0
self.last_saved = -1
self.requests = LifoQueue()
t = Thread(name='save-thread', target=self.run)
t.daemon = True
t.start()
self.status_widget = w = SaveWidget(parent)
self.start_save.connect(w.start, type=Qt.QueuedConnection)
self.save_done.connect(w.stop, type=Qt.QueuedConnection)
def schedule(self, tdir, container):
self.count += 1
self.requests.put((self.count, tdir, container))
def run(self):
while True:
x = self.requests.get()
if x is None:
self.requests.task_done()
self.__empty_queue()
break
try:
count, tdir, container = x
self.process_save(count, tdir, container)
except:
import traceback
traceback.print_exc()
finally:
self.requests.task_done()
def __empty_queue(self):
' Only to be used during shutdown '
while True:
try:
self.requests.get_nowait()
except Empty:
break
else:
self.requests.task_done()
def process_save(self, count, tdir, container):
if count <= self.last_saved:
shutil.rmtree(tdir, ignore_errors=True)
return
self.last_saved = count
self.start_save.emit()
try:
self.do_save(tdir, container)
except:
import traceback
self.report_error.emit(traceback.format_exc())
self.save_done.emit()
def do_save(self, tdir, container):
temp = None
try:
path = container.path_to_ebook
temp = PersistentTemporaryFile(prefix=('_' if iswindows else '.'),
suffix=os.path.splitext(path)[1],
dir=os.path.dirname(path))
temp.close()
temp = temp.name
container.commit(temp)
atomic_rename(temp, path)
finally:
if temp and os.path.exists(temp):
os.remove(temp)
shutil.rmtree(tdir, ignore_errors=True)
@property
def has_tasks(self):
return bool(self.requests.unfinished_tasks)
def wait(self, timeout=30):
if timeout is None:
self.requests.join()
else:
try:
join_with_timeout(self.requests, timeout)
except RuntimeError:
return False
return True
def shutdown(self):
self.requests.put(None)
