def add_element(self, elem): "Add an element to the datastore." self.db.store(elem) # If the element is a node, add it to the appropriate geodoc. ns = elem.namespace backreference = make_backreference(ns, elem.id) if self.verbose: increment_stats(ns) # Do element-specific processing. if ns == C.NODE: # Add the element to the appropriate geodoc. self.geotable.add(elem) elif ns == C.WAY: # Backlink referenced nodes to the current way. for (rstatus, node_or_key) in \ self.db.fetch_keys(C.NODE, map(str, elem[C.NODES])): if rstatus: node = node_or_key else: node = new_osm_element(C.NODE, node_or_key) node[C.REFERENCES].add(backreference) self.db.store(node) elif ns == C.RELATION: # If the element is a relation, backlink referenced ways & # relations. def _retrieve(selector, members): return [ str(mref) for (mref, mrole, mtype) in members if mtype == selector ] members = elem[C.MEMBERS] elements = [] for ns in [C.NODE, C.WAY, C.RELATIONS]: elements.append((ns, _retrieve(ns, members))) for (ns, refs) in elements: if len(refs) == 0: continue for (rstatus, node_or_key) in self.db.fetch_keys(ns, refs): # Retrieve all elements referenced by the relation. if rstatus: elem = node_or_key else: elem = new_osm_element(ns, node_or_key) # Add a backreference to the element being # referenced by this relation. elem[C.REFERENCES].add(backreference) self.db.store(elem)
def add_element(self, elem): "Add an element to the datastore." self.db.store(elem) # If the element is a node, add it to the appropriate geodoc. ns = elem.namespace backreference = make_backreference(ns, elem.id) if self.verbose: increment_stats(ns) # Do element-specific processing. if ns == C.NODE: # Add the element to the appropriate geodoc. self.geotable.add(elem) elif ns == C.WAY: # Backlink referenced nodes to the current way. for (rstatus, node_or_key) in self.db.fetch_keys(C.NODE, map(str, elem[C.NODES])): if rstatus: node = node_or_key else: node = new_osm_element(C.NODE, node_or_key) node[C.REFERENCES].add(backreference) self.db.store(node) elif ns == C.RELATION: # If the element is a relation, backlink referenced ways & # relations. def _retrieve(selector, members): return [str(mref) for (mref, mrole, mtype) in members if mtype == selector] members = elem[C.MEMBERS] elements = [] for ns in [C.NODE, C.WAY, C.RELATIONS]: elements.append((ns, _retrieve(ns, members))) for (ns, refs) in elements: if len(refs) == 0: continue for (rstatus, node_or_key) in self.db.fetch_keys(ns, refs): # Retrieve all elements referenced by the relation. if rstatus: elem = node_or_key else: elem = new_osm_element(ns, node_or_key) # Add a backreference to the element being # referenced by this relation. elem[C.REFERENCES].add(backreference) self.db.store(elem)
def test_datastore_slab_inline_fetch(datastore): "Verify that elements in a slab are fetched." _slab_key = 'NL8' _slab_start = __PER_SLAB # Create a slab. slab = [] slabkeys = range(_slab_start, _slab_start + __PER_SLAB) for key in slabkeys: sk = str(key) if key % 2 == 0: n = O.new_osm_element(C.NODE, sk) slab.append((C.SLAB_INLINE, n)) insert_key(_slab_key, datastore.encode(slab)) c = 0 i = 0 for key in slabkeys: if key % 2 == 0: se,sn = slab[i] n = datastore.fetch(C.NODE, str(key)) assert n == sn i += 1 else: v = datastore.fetch(C.NODE, str(key)) assert v is None c = c + 1
def test_geokeys(): "Test geo hash keys returned for various coordinates." init_geohash(_GHKEYLENGTH, _SCALEFACTOR) expected = [ (0.0, 0.0, 's0000'), (89, 0.0, 'upb42'), (89.999999999999992, 0.0, 'upbpb'), # Max lat supported. (-90, 0.0, 'h0000'), (-90, -180, '00000'), (-90, +180, '00000'), (-90, +90, 'n0000'), (-90, -90, '40000'), (-45, -45, '70000'), (-45, 45, 'm0000'), (45, 45, 'v0000'), (45, -45, 'g0000') ] for (lat, lon, ghkey) in expected: elem = new_osm_element(C.NODE, '0') elem[C.LAT] = lat * _SCALEFACTOR elem[C.LON] = lon * _SCALEFACTOR res = geohash_key_for_element(elem) assert res == ghkey
def retrieve_slab(self, namespace, slabkey): """Return a slab of elements.""" db = self._get_connection() wirebits = db.get(slabkey) if wirebits is None: return None slab = [] for (st, kv) in self.decode(wirebits): if st == C.SLAB_NOT_PRESENT: continue if st == C.SLAB_INDIRECT: elem = self.retrieve_element(namespace, kv) assert elem is not None, "Missing indirect element" elif st == C.SLAB_INLINE: elem = new_osm_element(namespace, kv[C.ID]) elem.from_mapping(kv) else: assert False, "Unknown status %d" % status slab.append((elem.id, elem)) return slab
def test_datastore_slab_inline_fetch(datastore): "Verify that elements in a slab are fetched." _slab_key = 'NL8' _slab_start = __PER_SLAB # Create a slab. slab = [] slabkeys = range(_slab_start, _slab_start + __PER_SLAB) for key in slabkeys: sk = str(key) if key % 2 == 0: n = O.new_osm_element(C.NODE, sk) slab.append((C.SLAB_INLINE, n)) insert_key(_slab_key, datastore.encode(slab)) c = 0 i = 0 for key in slabkeys: if key % 2 == 0: se, sn = slab[i] n = datastore.fetch(C.NODE, str(key)) assert n == sn i += 1 else: v = datastore.fetch(C.NODE, str(key)) assert v is None c = c + 1
def retrieve_slab(self, namespace, slabkey): """Return a slab of elements.""" db = self._get_connection() wirebits = db.get(slabkey) if wirebits is None: return None slab = [] for (st, kv) in self.decode(wirebits): if st == C.SLAB_NOT_PRESENT: continue if st == C.SLAB_INDIRECT: elem = self.retrieve_element(namespace, kv) assert elem is not None, "Missing indirect element" elif st == C.SLAB_INLINE: elem = new_osm_element(namespace, kv[C.ID]) elem.from_mapping(kv) else: assert False, "Unknown status %d" % status slab.append((elem.id, elem)) return slab
def _make_osm_iterator(config, f): "Return an iterator parsing the <osm> format" scalefactor = config.getint(C.DATASTORE, C.SCALE_FACTOR) parser = iter(iterparse(f, events=('start', 'end'))) event, root = parser.next() if root.tag != u'osm': raise ValueError, "Unexpected root tag: %s" % root.tag depth = 0 doc = None ignored_elements = ['bound', 'bounds'] processed_elements = ('changeset', 'node', 'way', 'relation') # Parse the input file. for event, elem in parser: element_name = elem.tag if element_name in ignored_elements: continue if event == 'start': if element_name in processed_elements: assert depth == 0 # Start of the element. Copy 'standard' attributes, # translating them to native values where possible. doc = new_osm_element(element_name.lower(), elem.get('id')) for k,v in elem.items(): if k == 'visible': v = bool(v) elif k == 'version' or k == 'uid': v = int(v) elif k == 'lat' or k == 'lon': v = encode_coordinate(v) doc[k] = v elif element_name == 'tag': # Each 'tag' has a key/value associated with it. doc.setdefault('tags', {})[elem.get('k')] = elem.get('v') elif element_name == 'nd': # <nd> elements contain references. doc['nodes'].add(int(elem.get('ref'))) elif element_name == 'member': # Collect the list of (ref, role, type) tuples. doc.setdefault('members', []).append((elem.get('ref'), elem.get('role'), elem.get('type'))) depth = depth + 1 elif event == 'end': depth = depth - 1 if depth == 0: yield doc # Return a complete element to the caller. root.clear() # Keep memory usage down.
def test_datastore_direct_fetch(datastore): "Verify that directly fetchable elements can be read." _direct_key = 'Gs0000' _direct_val = O.new_osm_element(C.GEODOC, _direct_key[1:]) insert_key(_direct_key, O.encode_json(_direct_val)) v = datastore.fetch(C.GEODOC, _direct_key[1:]) assert v == _direct_val
def test_datastore_direct_fetch(datastore): "Verify that directly fetchable elements can be read." _direct_key = 'Gs0000' _direct_val = O.new_osm_element(C.GEODOC, _direct_key[1:]) insert_key(_direct_key, O.encode_json(_direct_val)) v = datastore.fetch(C.GEODOC, _direct_key[1:]) assert v == _direct_val
def _make_osm_iterator(config, f): "Return an iterator parsing the <osm> format" scalefactor = config.getint(C.DATASTORE, C.SCALE_FACTOR) parser = iter(iterparse(f, events=('start', 'end'))) event, root = parser.next() if root.tag != u'osm': raise ValueError, "Unexpected root tag: %s" % root.tag depth = 0 doc = None ignored_elements = ['bound', 'bounds'] processed_elements = ('changeset', 'node', 'way', 'relation') # Parse the input file. for event, elem in parser: element_name = elem.tag if element_name in ignored_elements: continue if event == 'start': if element_name in processed_elements: assert depth == 0 # Start of the element. Copy 'standard' attributes, # translating them to native values where possible. doc = new_osm_element(element_name.lower(), elem.get('id')) for k, v in elem.items(): if k == 'visible': v = bool(v) elif k == 'version' or k == 'uid': v = int(v) elif k == 'lat' or k == 'lon': v = encode_coordinate(v) doc[k] = v elif element_name == 'tag': # Each 'tag' has a key/value associated with it. doc.setdefault('tags', {})[elem.get('k')] = elem.get('v') elif element_name == 'nd': # <nd> elements contain references. doc['nodes'].add(int(elem.get('ref'))) elif element_name == 'member': # Collect the list of (ref, role, type) tuples. doc.setdefault('members', []).append( (elem.get('ref'), elem.get('role'), elem.get('type'))) depth = depth + 1 elif event == 'end': depth = depth - 1 if depth == 0: yield doc # Return a complete element to the caller. root.clear() # Keep memory usage down.
def _write_geodoc(self, key, nodegroup): "Merge in a group of nodes into a geodoc." assert isinstance(nodegroup, NodeGroup) geodoc = self.db.retrieve_element(C.GEODOC, key) if geodoc is None: # New document. geodoc = new_osm_element(C.GEODOC, key) nodegroup.update(geodoc[C.NODES]) geodoc[C.NODES] = nodegroup.aslist() self.db.store_element(C.GEODOC, key, geodoc)
def test_datastore_write_element(datastore): "Test the store_element() entry point." _geodoc_key = 'Gs0000' _geodoc_val = O.new_osm_element(C.GEODOC, _geodoc_key[1:]) datastore.store_element(C.GEODOC, _geodoc_key[1:], _geodoc_val) v = retrieve_key(_geodoc_key) assert v == O.encode_json(_geodoc_val)
def test_datastore_write_element(datastore): "Test the store_element() entry point." _geodoc_key = 'Gs0000' _geodoc_val = O.new_osm_element(C.GEODOC, _geodoc_key[1:]) datastore.store_element(C.GEODOC, _geodoc_key[1:], _geodoc_val) v = retrieve_key(_geodoc_key) assert v == O.encode_json(_geodoc_val)
def test_new_way(config): "Test creation of a <way> element." O.init_osm_factory(config) wayid = '42' w = O.new_osm_element(C.WAY, wayid) # Check the "id", NODES and REFERENCES attributes. assert w.id == str(wayid) assert w[C.REFERENCES] == set() assert w[C.NODES] == set()
def test_new_relation(config): "Test creation of a <relation> element." O.init_osm_factory(config) relid = '42' r = O.new_osm_element(C.RELATION, relid) # Check the "id", MEMBER and REFERENCES attributes. assert r.id == str(relid) assert r[C.REFERENCES] == set() assert r[C.MEMBERS] == []
def test_new_relation(config): "Test creation of a <relation> element." O.init_osm_factory(config) relid = '42' r = O.new_osm_element(C.RELATION, relid) # Check the "id", MEMBER and REFERENCES attributes. assert r.id == str(relid) assert r[C.REFERENCES] == set() assert r[C.MEMBERS] == []
def test_new_way(config): "Test creation of a <way> element." O.init_osm_factory(config) wayid = '42' w = O.new_osm_element(C.WAY, wayid) # Check the "id", NODES and REFERENCES attributes. assert w.id == str(wayid) assert w[C.REFERENCES] == set() assert w[C.NODES] == set()
def test_new_node(config): "Test the creation a <node> element." O.init_osm_factory(config) nodeid = '42' n = O.new_osm_element(C.NODE, nodeid) # Check the 'id' field. assert n.id == str(nodeid) # Check that C.REFERENCES field exists, and is an empty set. assert n[C.REFERENCES] == set()
def test_new_node(config): "Test the creation a <node> element." O.init_osm_factory(config) nodeid = '42' n = O.new_osm_element(C.NODE, nodeid) # Check the 'id' field. assert n.id == str(nodeid) # Check that C.REFERENCES field exists, and is an empty set. assert n[C.REFERENCES] == set()
def test_new_geodoc(config): "Test the creation of a geodoc element." O.init_osm_factory(config) georegion = 'szmyg' # lat, long == 42, 42 g = O.new_osm_element(C.GEODOC, georegion) # Check the "id" field. assert g.id == georegion assert g[C.NODES] == set() bbox = g[C.BBOX] assert set(bbox.keys()) == set(['n', 's', 'e', 'w'])
def test_new_geodoc(config): "Test the creation of a geodoc element." O.init_osm_factory(config) georegion = 'szmyg' # lat, long == 42, 42 g = O.new_osm_element(C.GEODOC, georegion) # Check the "id" field. assert g.id == georegion assert g[C.NODES] == set() bbox = g[C.BBOX] assert set(bbox.keys()) == set(['n', 's', 'e', 'w'])
def __init__(self, config, usethreads=False, writeback=False): "Initialize the datastore." self.conndb = {} DatastoreBase.__init__(self, config, usethreads, writeback) dbhosts = config.get(C.MEMBASE, C.DBHOST) dbport = config.get(C.MEMBASE, C.DBPORT) self.membasehosts = [h + ':' + dbport for h in dbhosts.split()] threads = [threading.currentThread()] if usethreads: threads.extend(self.threads) self.register_threads(threads) if writeback: # Store slab configuration information for subsequent # retrieval by the front end. slabconfig = new_osm_element(C.DATASTORE_CONFIG, C.CFGSLAB) for k in DatastoreMembase.SLAB_CONFIGURATION_KEYS: slabconfig[k] = config.get(C.DATASTORE, k) slabconfig[C.CONFIGURATION_SCHEMA_VERSION] = C.CFGVERSION self.slabconfig = slabconfig else: # Read slab configuration information from the data store. self.slabconfig = slabconfig = \ self.retrieve_element(C.DATASTORE_CONFIG, C.CFGSLAB) if slabconfig is not None: schema_version = slabconfig.get(C.CONFIGURATION_SCHEMA_VERSION) if schema_version != C.CFGVERSION: raise ValueError, \ "Datastore schema version mismatch: expected %s, " \ "actual %s." % \ (str(C.CFGVERSION), str(schema_version)) for (k,v) in slabconfig.items(): if k in DatastoreMembase.SLAB_CONFIGURATION_KEYS: config.set(C.DATASTORE, k, v) else: raise ValueError, \ "Datastore is missing configuration information."
def retrieve_element(self, namespace, key): """Return the element for a key. Parameters: namespace - namespace for element. key - the key to retrieve. """ dskey = namespace[0].upper() + key db = self._get_connection() wirebits = db.get(dskey) if wirebits is None: return None n = new_osm_element(namespace, key) n.from_mapping(self.decode(wirebits)) return n
def retrieve_element(self, namespace, key): """Return the element for a key. Parameters: namespace - namespace for element. key - the key to retrieve. """ dskey = namespace[0].upper() + key db = self._get_connection() wirebits = db.get(dskey) if wirebits is None: return None n = new_osm_element(namespace, key) n.from_mapping(self.decode(wirebits)) return n
def __init__(self, config, usethreads=False, writeback=False): "Initialize the datastore." self.conndb = {} DatastoreBase.__init__(self, config, usethreads, writeback) dbhosts = config.get(C.MEMBASE, C.DBHOST) dbport = config.get(C.MEMBASE, C.DBPORT) self.membasehosts = [h + ':' + dbport for h in dbhosts.split()] threads = [threading.currentThread()] if usethreads: threads.extend(self.threads) self.register_threads(threads) if writeback: # Store slab configuration information for subsequent # retrieval by the front end. slabconfig = new_osm_element(C.DATASTORE_CONFIG, C.CFGSLAB) for k in DatastoreMembase.SLAB_CONFIGURATION_KEYS: slabconfig[k] = config.get(C.DATASTORE, k) slabconfig[C.CONFIGURATION_SCHEMA_VERSION] = C.CFGVERSION self.slabconfig = slabconfig else: # Read slab configuration information from the data store. self.slabconfig = slabconfig = \ self.retrieve_element(C.DATASTORE_CONFIG, C.CFGSLAB) if slabconfig is not None: schema_version = slabconfig.get(C.CONFIGURATION_SCHEMA_VERSION) if schema_version != C.CFGVERSION: raise ValueError, \ "Datastore schema version mismatch: expected %s, " \ "actual %s." % \ (str(C.CFGVERSION), str(schema_version)) for (k, v) in slabconfig.items(): if k in DatastoreMembase.SLAB_CONFIGURATION_KEYS: config.set(C.DATASTORE, k, v) else: raise ValueError, \ "Datastore is missing configuration information."