Пример #1
0
    def __init__(self, config, options, db):
        '''Initialize the table.

        Keyword arguments:
        config        - A ConfigParser instance.
        options       - An optparse.OptionParser structure.
        db            - A DB object supporting 'get()' and 'store()'
                        methods.
        '''
        self.geodb = collections.defaultdict(NodeGroup)
        self.db = db

        lrusize = config.getint(C.DATASTORE, C.GEODOC_LRU_SIZE)
        self.lru = BoundedLRUBuffer(bound=lrusize, callback=self._cb)

        if options.nothreading:
            nthreads = 0
        else:
            nthreads = config.getint(C.DATASTORE, C.GEODOC_LRU_THREADS)
        self.nthreads = max(0, nthreads)
        if self.nthreads:
            self.wrthreads = []
            self.wrqueue = Queue(self.nthreads)
            self.wrcond = threading.Condition()
            self.wrpending = []
            for n in range(self.nthreads):
                t = threading.Thread(target=self._worker)
                t.name = "GeoWB-%d" % n
                t.daemon = True
                self.wrthreads.append(t)
                t.start()

            db.register_threads(self.wrthreads)
Пример #2
0
class GeoGroupTable:
    '''Group OSM nodes by their geographical coordinates.

    The coordinates of the globe are partitioned into disjoint areas.
    Each partition is named by the geohash code of its (n,w) corner.

    Grouping of nodes is implemented by restricting the length of
    the geohash codes used.
    '''

    def __init__(self, config, options, db):
        '''Initialize the table.

        Keyword arguments:
        config        - A ConfigParser instance.
        options       - An optparse.OptionParser structure.
        db            - A DB object supporting 'get()' and 'store()'
                        methods.
        '''
        self.geodb = collections.defaultdict(NodeGroup)
        self.db = db

        lrusize = config.getint(C.DATASTORE, C.GEODOC_LRU_SIZE)
        self.lru = BoundedLRUBuffer(bound=lrusize, callback=self._cb)

        if options.nothreading:
            nthreads = 0
        else:
            nthreads = config.getint(C.DATASTORE, C.GEODOC_LRU_THREADS)
        self.nthreads = max(0, nthreads)
        if self.nthreads:
            self.wrthreads = []
            self.wrqueue = Queue(self.nthreads)
            self.wrcond = threading.Condition()
            self.wrpending = []
            for n in range(self.nthreads):
                t = threading.Thread(target=self._worker)
                t.name = "GeoWB-%d" % n
                t.daemon = True
                self.wrthreads.append(t)
                t.start()

            db.register_threads(self.wrthreads)

    def _cb(self, key, value):
        "Callback called when an LRU item is ejected."
        nodeset = self.geodb.pop(key)
        if self.nthreads:       # Defer processing to a worker thread.
            self.wrqueue.put((key, nodeset))
        else:                   # Synchronous operation.
            self._write_geodoc(key, nodeset)

    def _worker(self):
        "Helper method, used by worker threads."
        while True:
            # Retrieve a work item.
            v = self.wrqueue.get()
            if v is None:     # Exit the thread.
                self.wrqueue.task_done()
                return

            # Unpack the work item.
            key, nodeset = v

            # Mark the item as "I/O in progress".
            with self.wrcond:
                while key in self.wrpending:
                    self.wrcond.wait()

                assert key not in self.wrpending
                self.wrpending.append(key)

            # Process this node set.
            self._write_geodoc(key, nodeset)

            # Remove the "I/O in progress" marker.
            with self.wrcond:
                assert key in self.wrpending
                self.wrpending.remove(key)
                self.wrcond.notifyAll()

            self.wrqueue.task_done()

    def _write_geodoc(self, key, nodegroup):
        "Merge in a group of nodes into a geodoc."
        assert isinstance(nodegroup, NodeGroup)

        geodoc = self.db.retrieve_element(C.GEODOC, key)
        if geodoc is None:      # New document.
            geodoc = new_osm_element(C.GEODOC, key)
        nodegroup.update(geodoc[C.NODES])
        geodoc[C.NODES] = nodegroup.aslist()
        self.db.store_element(C.GEODOC, key, geodoc)

    def add(self, elem):
        '''Add information about a node 'elem' to the geo table.

        Usage:
        >>> gt = GeoGroupTable()
        >>> gt = gt.add(elem)

        The node 'elem' should have a 'lat' and 'lon' fields that
        encode its latitude and longitude respectively.  The 'id'
        field specifies the node's "id".
        '''

        assert elem.namespace == C.NODE, "elem is not a node: %s" % str(elem)

        # Determine the geo-key for the node.
        ghkey = geohash_key_for_element(elem)
        # Retrieve the partition covering this location.
        ghdoc = self.geodb[ghkey]

        elemid = elem.id
        if elemid not in ghdoc:
            ghdoc.add(elem)
            self.lru[ghkey] = ghdoc

    def flush(self):
        "Wait pending I/Os"

        # Flush items from the LRU.
        self.lru.flush()

        if self.nthreads:
            # Wait for the work queue to drain.
            self.wrqueue.join()