def __init__(self, config, options, db):
'''Initialize the table.
Keyword arguments:
config - A ConfigParser instance.
options - An optparse.OptionParser structure.
db - A DB object supporting 'get()' and 'store()'
methods.
'''
self.geodb = collections.defaultdict(NodeGroup)
self.db = db
lrusize = config.getint(C.DATASTORE, C.GEODOC_LRU_SIZE)
self.lru = BoundedLRUBuffer(bound=lrusize, callback=self._cb)
if options.nothreading:
nthreads = 0
else:
nthreads = config.getint(C.DATASTORE, C.GEODOC_LRU_THREADS)
self.nthreads = max(0, nthreads)
if self.nthreads:
self.wrthreads = []
self.wrqueue = Queue(self.nthreads)
self.wrcond = threading.Condition()
self.wrpending = []
for n in range(self.nthreads):
t = threading.Thread(target=self._worker)
t.name = "GeoWB-%d" % n
t.daemon = True
self.wrthreads.append(t)
t.start()
db.register_threads(self.wrthreads)
class GeoGroupTable:
'''Group OSM nodes by their geographical coordinates.
The coordinates of the globe are partitioned into disjoint areas.
Each partition is named by the geohash code of its (n,w) corner.
Grouping of nodes is implemented by restricting the length of
the geohash codes used.
'''
def __init__(self, config, options, db):
'''Initialize the table.
Keyword arguments:
config - A ConfigParser instance.
options - An optparse.OptionParser structure.
db - A DB object supporting 'get()' and 'store()'
methods.
'''
self.geodb = collections.defaultdict(NodeGroup)
self.db = db
lrusize = config.getint(C.DATASTORE, C.GEODOC_LRU_SIZE)
self.lru = BoundedLRUBuffer(bound=lrusize, callback=self._cb)
if options.nothreading:
nthreads = 0
else:
nthreads = config.getint(C.DATASTORE, C.GEODOC_LRU_THREADS)
self.nthreads = max(0, nthreads)
if self.nthreads:
self.wrthreads = []
self.wrqueue = Queue(self.nthreads)
self.wrcond = threading.Condition()
self.wrpending = []
for n in range(self.nthreads):
t = threading.Thread(target=self._worker)
t.name = "GeoWB-%d" % n
t.daemon = True
self.wrthreads.append(t)
t.start()
db.register_threads(self.wrthreads)
def _cb(self, key, value):
"Callback called when an LRU item is ejected."
nodeset = self.geodb.pop(key)
if self.nthreads: # Defer processing to a worker thread.
self.wrqueue.put((key, nodeset))
else: # Synchronous operation.
self._write_geodoc(key, nodeset)
def _worker(self):
"Helper method, used by worker threads."
while True:
# Retrieve a work item.
v = self.wrqueue.get()
if v is None: # Exit the thread.
self.wrqueue.task_done()
return
# Unpack the work item.
key, nodeset = v
# Mark the item as "I/O in progress".
with self.wrcond:
while key in self.wrpending:
self.wrcond.wait()
assert key not in self.wrpending
self.wrpending.append(key)
# Process this node set.
self._write_geodoc(key, nodeset)
# Remove the "I/O in progress" marker.
with self.wrcond:
assert key in self.wrpending
self.wrpending.remove(key)
self.wrcond.notifyAll()
self.wrqueue.task_done()
def _write_geodoc(self, key, nodegroup):
"Merge in a group of nodes into a geodoc."
assert isinstance(nodegroup, NodeGroup)
geodoc = self.db.retrieve_element(C.GEODOC, key)
if geodoc is None: # New document.
geodoc = new_osm_element(C.GEODOC, key)
nodegroup.update(geodoc[C.NODES])
geodoc[C.NODES] = nodegroup.aslist()
self.db.store_element(C.GEODOC, key, geodoc)
def add(self, elem):
'''Add information about a node 'elem' to the geo table.
Usage:
>>> gt = GeoGroupTable()
>>> gt = gt.add(elem)
The node 'elem' should have a 'lat' and 'lon' fields that
encode its latitude and longitude respectively. The 'id'
field specifies the node's "id".
'''
assert elem.namespace == C.NODE, "elem is not a node: %s" % str(elem)
# Determine the geo-key for the node.
ghkey = geohash_key_for_element(elem)
# Retrieve the partition covering this location.
ghdoc = self.geodb[ghkey]
elemid = elem.id
if elemid not in ghdoc:
ghdoc.add(elem)
self.lru[ghkey] = ghdoc
def flush(self):
"Wait pending I/Os"
# Flush items from the LRU.
self.lru.flush()
if self.nthreads:
# Wait for the work queue to drain.
self.wrqueue.join()