def add_element(self, elem):
"Add an element to the datastore."
self.db.store(elem)
# If the element is a node, add it to the appropriate geodoc.
ns = elem.namespace
backreference = make_backreference(ns, elem.id)
if self.verbose:
increment_stats(ns)
# Do element-specific processing.
if ns == C.NODE:
# Add the element to the appropriate geodoc.
self.geotable.add(elem)
elif ns == C.WAY:
# Backlink referenced nodes to the current way.
for (rstatus, node_or_key) in \
self.db.fetch_keys(C.NODE, map(str, elem[C.NODES])):
if rstatus:
node = node_or_key
else:
node = new_osm_element(C.NODE, node_or_key)
node[C.REFERENCES].add(backreference)
self.db.store(node)
elif ns == C.RELATION:
# If the element is a relation, backlink referenced ways &
# relations.
def _retrieve(selector, members):
return [
str(mref) for (mref, mrole, mtype) in members
if mtype == selector
]
members = elem[C.MEMBERS]
elements = []
for ns in [C.NODE, C.WAY, C.RELATIONS]:
elements.append((ns, _retrieve(ns, members)))
for (ns, refs) in elements:
if len(refs) == 0:
continue
for (rstatus, node_or_key) in self.db.fetch_keys(ns, refs):
# Retrieve all elements referenced by the relation.
if rstatus:
elem = node_or_key
else:
elem = new_osm_element(ns, node_or_key)
# Add a backreference to the element being
# referenced by this relation.
elem[C.REFERENCES].add(backreference)
self.db.store(elem)
def add_element(self, elem):
"Add an element to the datastore."
self.db.store(elem)
# If the element is a node, add it to the appropriate geodoc.
ns = elem.namespace
backreference = make_backreference(ns, elem.id)
if self.verbose:
increment_stats(ns)
# Do element-specific processing.
if ns == C.NODE:
# Add the element to the appropriate geodoc.
self.geotable.add(elem)
elif ns == C.WAY:
# Backlink referenced nodes to the current way.
for (rstatus, node_or_key) in self.db.fetch_keys(C.NODE, map(str, elem[C.NODES])):
if rstatus:
node = node_or_key
else:
node = new_osm_element(C.NODE, node_or_key)
node[C.REFERENCES].add(backreference)
self.db.store(node)
elif ns == C.RELATION:
# If the element is a relation, backlink referenced ways &
# relations.
def _retrieve(selector, members):
return [str(mref) for (mref, mrole, mtype) in members if mtype == selector]
members = elem[C.MEMBERS]
elements = []
for ns in [C.NODE, C.WAY, C.RELATIONS]:
elements.append((ns, _retrieve(ns, members)))
for (ns, refs) in elements:
if len(refs) == 0:
continue
for (rstatus, node_or_key) in self.db.fetch_keys(ns, refs):
# Retrieve all elements referenced by the relation.
if rstatus:
elem = node_or_key
else:
elem = new_osm_element(ns, node_or_key)
# Add a backreference to the element being
# referenced by this relation.
elem[C.REFERENCES].add(backreference)
self.db.store(elem)
def test_datastore_slab_inline_fetch(datastore):
"Verify that elements in a slab are fetched."
_slab_key = 'NL8'
_slab_start = __PER_SLAB
# Create a slab.
slab = []
slabkeys = range(_slab_start, _slab_start + __PER_SLAB)
for key in slabkeys:
sk = str(key)
if key % 2 == 0:
n = O.new_osm_element(C.NODE, sk)
slab.append((C.SLAB_INLINE, n))
insert_key(_slab_key, datastore.encode(slab))
c = 0
i = 0
for key in slabkeys:
if key % 2 == 0:
se,sn = slab[i]
n = datastore.fetch(C.NODE, str(key))
assert n == sn
i += 1
else:
v = datastore.fetch(C.NODE, str(key))
assert v is None
c = c + 1
def test_geokeys():
"Test geo hash keys returned for various coordinates."
init_geohash(_GHKEYLENGTH, _SCALEFACTOR)
expected = [
(0.0, 0.0, 's0000'),
(89, 0.0, 'upb42'),
(89.999999999999992, 0.0, 'upbpb'), # Max lat supported.
(-90, 0.0, 'h0000'),
(-90, -180, '00000'),
(-90, +180, '00000'),
(-90, +90, 'n0000'),
(-90, -90, '40000'),
(-45, -45, '70000'),
(-45, 45, 'm0000'),
(45, 45, 'v0000'),
(45, -45, 'g0000')
]
for (lat, lon, ghkey) in expected:
elem = new_osm_element(C.NODE, '0')
elem[C.LAT] = lat * _SCALEFACTOR
elem[C.LON] = lon * _SCALEFACTOR
res = geohash_key_for_element(elem)
assert res == ghkey
def retrieve_slab(self, namespace, slabkey):
"""Return a slab of elements."""
db = self._get_connection()
wirebits = db.get(slabkey)
if wirebits is None:
return None
slab = []
for (st, kv) in self.decode(wirebits):
if st == C.SLAB_NOT_PRESENT:
continue
if st == C.SLAB_INDIRECT:
elem = self.retrieve_element(namespace, kv)
assert elem is not None, "Missing indirect element"
elif st == C.SLAB_INLINE:
elem = new_osm_element(namespace, kv[C.ID])
elem.from_mapping(kv)
else:
assert False, "Unknown status %d" % status
slab.append((elem.id, elem))
return slab
def test_datastore_slab_inline_fetch(datastore):
"Verify that elements in a slab are fetched."
_slab_key = 'NL8'
_slab_start = __PER_SLAB
# Create a slab.
slab = []
slabkeys = range(_slab_start, _slab_start + __PER_SLAB)
for key in slabkeys:
sk = str(key)
if key % 2 == 0:
n = O.new_osm_element(C.NODE, sk)
slab.append((C.SLAB_INLINE, n))
insert_key(_slab_key, datastore.encode(slab))
c = 0
i = 0
for key in slabkeys:
if key % 2 == 0:
se, sn = slab[i]
n = datastore.fetch(C.NODE, str(key))
assert n == sn
i += 1
else:
v = datastore.fetch(C.NODE, str(key))
assert v is None
c = c + 1
def retrieve_slab(self, namespace, slabkey):
"""Return a slab of elements."""
db = self._get_connection()
wirebits = db.get(slabkey)
if wirebits is None:
return None
slab = []
for (st, kv) in self.decode(wirebits):
if st == C.SLAB_NOT_PRESENT:
continue
if st == C.SLAB_INDIRECT:
elem = self.retrieve_element(namespace, kv)
assert elem is not None, "Missing indirect element"
elif st == C.SLAB_INLINE:
elem = new_osm_element(namespace, kv[C.ID])
elem.from_mapping(kv)
else:
assert False, "Unknown status %d" % status
slab.append((elem.id, elem))
return slab
def _make_osm_iterator(config, f):
"Return an iterator parsing the <osm> format"
scalefactor = config.getint(C.DATASTORE, C.SCALE_FACTOR)
parser = iter(iterparse(f, events=('start', 'end')))
event, root = parser.next()
if root.tag != u'osm':
raise ValueError, "Unexpected root tag: %s" % root.tag
depth = 0
doc = None
ignored_elements = ['bound', 'bounds']
processed_elements = ('changeset', 'node', 'way', 'relation')
# Parse the input file.
for event, elem in parser:
element_name = elem.tag
if element_name in ignored_elements:
continue
if event == 'start':
if element_name in processed_elements:
assert depth == 0
# Start of the element. Copy 'standard' attributes,
# translating them to native values where possible.
doc = new_osm_element(element_name.lower(), elem.get('id'))
for k,v in elem.items():
if k == 'visible':
v = bool(v)
elif k == 'version' or k == 'uid':
v = int(v)
elif k == 'lat' or k == 'lon':
v = encode_coordinate(v)
doc[k] = v
elif element_name == 'tag':
# Each 'tag' has a key/value associated with it.
doc.setdefault('tags', {})[elem.get('k')] = elem.get('v')
elif element_name == 'nd':
# <nd> elements contain references.
doc['nodes'].add(int(elem.get('ref')))
elif element_name == 'member':
# Collect the list of (ref, role, type) tuples.
doc.setdefault('members', []).append((elem.get('ref'),
elem.get('role'),
elem.get('type')))
depth = depth + 1
elif event == 'end':
depth = depth - 1
if depth == 0:
yield doc # Return a complete element to the caller.
root.clear() # Keep memory usage down.
def test_datastore_direct_fetch(datastore):
"Verify that directly fetchable elements can be read."
_direct_key = 'Gs0000'
_direct_val = O.new_osm_element(C.GEODOC, _direct_key[1:])
insert_key(_direct_key, O.encode_json(_direct_val))
v = datastore.fetch(C.GEODOC, _direct_key[1:])
assert v == _direct_val
def test_datastore_direct_fetch(datastore):
"Verify that directly fetchable elements can be read."
_direct_key = 'Gs0000'
_direct_val = O.new_osm_element(C.GEODOC, _direct_key[1:])
insert_key(_direct_key, O.encode_json(_direct_val))
v = datastore.fetch(C.GEODOC, _direct_key[1:])
assert v == _direct_val
def _make_osm_iterator(config, f):
"Return an iterator parsing the <osm> format"
scalefactor = config.getint(C.DATASTORE, C.SCALE_FACTOR)
parser = iter(iterparse(f, events=('start', 'end')))
event, root = parser.next()
if root.tag != u'osm':
raise ValueError, "Unexpected root tag: %s" % root.tag
depth = 0
doc = None
ignored_elements = ['bound', 'bounds']
processed_elements = ('changeset', 'node', 'way', 'relation')
# Parse the input file.
for event, elem in parser:
element_name = elem.tag
if element_name in ignored_elements:
continue
if event == 'start':
if element_name in processed_elements:
assert depth == 0
# Start of the element. Copy 'standard' attributes,
# translating them to native values where possible.
doc = new_osm_element(element_name.lower(), elem.get('id'))
for k, v in elem.items():
if k == 'visible':
v = bool(v)
elif k == 'version' or k == 'uid':
v = int(v)
elif k == 'lat' or k == 'lon':
v = encode_coordinate(v)
doc[k] = v
elif element_name == 'tag':
# Each 'tag' has a key/value associated with it.
doc.setdefault('tags', {})[elem.get('k')] = elem.get('v')
elif element_name == 'nd':
# <nd> elements contain references.
doc['nodes'].add(int(elem.get('ref')))
elif element_name == 'member':
# Collect the list of (ref, role, type) tuples.
doc.setdefault('members', []).append(
(elem.get('ref'), elem.get('role'), elem.get('type')))
depth = depth + 1
elif event == 'end':
depth = depth - 1
if depth == 0:
yield doc # Return a complete element to the caller.
root.clear() # Keep memory usage down.
def _write_geodoc(self, key, nodegroup):
"Merge in a group of nodes into a geodoc."
assert isinstance(nodegroup, NodeGroup)
geodoc = self.db.retrieve_element(C.GEODOC, key)
if geodoc is None: # New document.
geodoc = new_osm_element(C.GEODOC, key)
nodegroup.update(geodoc[C.NODES])
geodoc[C.NODES] = nodegroup.aslist()
self.db.store_element(C.GEODOC, key, geodoc)
def test_datastore_write_element(datastore):
"Test the store_element() entry point."
_geodoc_key = 'Gs0000'
_geodoc_val = O.new_osm_element(C.GEODOC, _geodoc_key[1:])
datastore.store_element(C.GEODOC, _geodoc_key[1:], _geodoc_val)
v = retrieve_key(_geodoc_key)
assert v == O.encode_json(_geodoc_val)
def test_datastore_write_element(datastore):
"Test the store_element() entry point."
_geodoc_key = 'Gs0000'
_geodoc_val = O.new_osm_element(C.GEODOC, _geodoc_key[1:])
datastore.store_element(C.GEODOC, _geodoc_key[1:], _geodoc_val)
v = retrieve_key(_geodoc_key)
assert v == O.encode_json(_geodoc_val)
def test_new_way(config):
"Test creation of a <way> element."
O.init_osm_factory(config)
wayid = '42'
w = O.new_osm_element(C.WAY, wayid)
# Check the "id", NODES and REFERENCES attributes.
assert w.id == str(wayid)
assert w[C.REFERENCES] == set()
assert w[C.NODES] == set()
def test_new_relation(config):
"Test creation of a <relation> element."
O.init_osm_factory(config)
relid = '42'
r = O.new_osm_element(C.RELATION, relid)
# Check the "id", MEMBER and REFERENCES attributes.
assert r.id == str(relid)
assert r[C.REFERENCES] == set()
assert r[C.MEMBERS] == []
def test_new_relation(config):
"Test creation of a <relation> element."
O.init_osm_factory(config)
relid = '42'
r = O.new_osm_element(C.RELATION, relid)
# Check the "id", MEMBER and REFERENCES attributes.
assert r.id == str(relid)
assert r[C.REFERENCES] == set()
assert r[C.MEMBERS] == []
def test_new_way(config):
"Test creation of a <way> element."
O.init_osm_factory(config)
wayid = '42'
w = O.new_osm_element(C.WAY, wayid)
# Check the "id", NODES and REFERENCES attributes.
assert w.id == str(wayid)
assert w[C.REFERENCES] == set()
assert w[C.NODES] == set()
def test_new_node(config):
"Test the creation a <node> element."
O.init_osm_factory(config)
nodeid = '42'
n = O.new_osm_element(C.NODE, nodeid)
# Check the 'id' field.
assert n.id == str(nodeid)
# Check that C.REFERENCES field exists, and is an empty set.
assert n[C.REFERENCES] == set()
def test_new_node(config):
"Test the creation a <node> element."
O.init_osm_factory(config)
nodeid = '42'
n = O.new_osm_element(C.NODE, nodeid)
# Check the 'id' field.
assert n.id == str(nodeid)
# Check that C.REFERENCES field exists, and is an empty set.
assert n[C.REFERENCES] == set()
def test_new_geodoc(config):
"Test the creation of a geodoc element."
O.init_osm_factory(config)
georegion = 'szmyg' # lat, long == 42, 42
g = O.new_osm_element(C.GEODOC, georegion)
# Check the "id" field.
assert g.id == georegion
assert g[C.NODES] == set()
bbox = g[C.BBOX]
assert set(bbox.keys()) == set(['n', 's', 'e', 'w'])
def test_new_geodoc(config):
"Test the creation of a geodoc element."
O.init_osm_factory(config)
georegion = 'szmyg' # lat, long == 42, 42
g = O.new_osm_element(C.GEODOC, georegion)
# Check the "id" field.
assert g.id == georegion
assert g[C.NODES] == set()
bbox = g[C.BBOX]
assert set(bbox.keys()) == set(['n', 's', 'e', 'w'])
def __init__(self, config, usethreads=False, writeback=False):
"Initialize the datastore."
self.conndb = {}
DatastoreBase.__init__(self, config, usethreads, writeback)
dbhosts = config.get(C.MEMBASE, C.DBHOST)
dbport = config.get(C.MEMBASE, C.DBPORT)
self.membasehosts = [h + ':' + dbport for h in dbhosts.split()]
threads = [threading.currentThread()]
if usethreads:
threads.extend(self.threads)
self.register_threads(threads)
if writeback:
# Store slab configuration information for subsequent
# retrieval by the front end.
slabconfig = new_osm_element(C.DATASTORE_CONFIG, C.CFGSLAB)
for k in DatastoreMembase.SLAB_CONFIGURATION_KEYS:
slabconfig[k] = config.get(C.DATASTORE, k)
slabconfig[C.CONFIGURATION_SCHEMA_VERSION] = C.CFGVERSION
self.slabconfig = slabconfig
else:
# Read slab configuration information from the data store.
self.slabconfig = slabconfig = \
self.retrieve_element(C.DATASTORE_CONFIG, C.CFGSLAB)
if slabconfig is not None:
schema_version = slabconfig.get(C.CONFIGURATION_SCHEMA_VERSION)
if schema_version != C.CFGVERSION:
raise ValueError, \
"Datastore schema version mismatch: expected %s, " \
"actual %s." % \
(str(C.CFGVERSION), str(schema_version))
for (k,v) in slabconfig.items():
if k in DatastoreMembase.SLAB_CONFIGURATION_KEYS:
config.set(C.DATASTORE, k, v)
else:
raise ValueError, \
"Datastore is missing configuration information."
def retrieve_element(self, namespace, key):
"""Return the element for a key.
Parameters:
namespace - namespace for element.
key - the key to retrieve.
"""
dskey = namespace[0].upper() + key
db = self._get_connection()
wirebits = db.get(dskey)
if wirebits is None:
return None
n = new_osm_element(namespace, key)
n.from_mapping(self.decode(wirebits))
return n
def retrieve_element(self, namespace, key):
"""Return the element for a key.
Parameters:
namespace - namespace for element.
key - the key to retrieve.
"""
dskey = namespace[0].upper() + key
db = self._get_connection()
wirebits = db.get(dskey)
if wirebits is None:
return None
n = new_osm_element(namespace, key)
n.from_mapping(self.decode(wirebits))
return n
def __init__(self, config, usethreads=False, writeback=False):
"Initialize the datastore."
self.conndb = {}
DatastoreBase.__init__(self, config, usethreads, writeback)
dbhosts = config.get(C.MEMBASE, C.DBHOST)
dbport = config.get(C.MEMBASE, C.DBPORT)
self.membasehosts = [h + ':' + dbport for h in dbhosts.split()]
threads = [threading.currentThread()]
if usethreads:
threads.extend(self.threads)
self.register_threads(threads)
if writeback:
# Store slab configuration information for subsequent
# retrieval by the front end.
slabconfig = new_osm_element(C.DATASTORE_CONFIG, C.CFGSLAB)
for k in DatastoreMembase.SLAB_CONFIGURATION_KEYS:
slabconfig[k] = config.get(C.DATASTORE, k)
slabconfig[C.CONFIGURATION_SCHEMA_VERSION] = C.CFGVERSION
self.slabconfig = slabconfig
else:
# Read slab configuration information from the data store.
self.slabconfig = slabconfig = \
self.retrieve_element(C.DATASTORE_CONFIG, C.CFGSLAB)
if slabconfig is not None:
schema_version = slabconfig.get(C.CONFIGURATION_SCHEMA_VERSION)
if schema_version != C.CFGVERSION:
raise ValueError, \
"Datastore schema version mismatch: expected %s, " \
"actual %s." % \
(str(C.CFGVERSION), str(schema_version))
for (k, v) in slabconfig.items():
if k in DatastoreMembase.SLAB_CONFIGURATION_KEYS:
config.set(C.DATASTORE, k, v)
else:
raise ValueError, \
"Datastore is missing configuration information."
