def __init__(self, db, collection, index_path=None, srid=None, fc=None, clear=False):
# pull collection metadata from the database if it exists
if clear:
GeoIndex(index_path, collection, _from_objectid, _to_objectid, srid=srid if srid else 4326).drop()
db[collection].drop()
db.geojson__meta.remove(collection)
self.collection_metadata = db.geojson__meta
self.meta = db.geojson__meta.find_one(collection) or {
'srid' : srid,
'index_path' : index_path,
'_id' : collection,
'properties' : {}
}
self.collection_metadata.save(self.meta)
if collection not in db.collection_names():
db.create_collection(collection)
if fc and not self.meta['srid'] and 'crs' in fc:
self.srid = fc['crs'] # however this means crs must conform to an integer srid. This is not preferred by the spec but necessary here.
# instantiate GeoIndex for collection
self.index = GeoIndex(index_path, collection, _from_objectid, _to_objectid, srid=srid if srid else 4326)
self.coll = db[collection]
if fc:
self.insert_features(fc, replace=True)
def test_uniquity(self):
ix = GeoIndex('./test_indices', 'test_unique', int, int)
ix.insert(1L, poly1)
self.assertTrue(ix.exists(1))
try:
self.assertRaises(IntegrityError, ix.insert(1, poly1))
except IntegrityError:
pass
try:
self.assertRaises(IntegrityError, ix.bulk_insert([(1,poly1), (1,poly1)]))
except IntegrityError:
pass
ix.drop()
def test_insert_one(self):
ix = GeoIndex('./test_indices', 'test_insert_one', int, int)
ix.insert(1L, poly1)
x = ix.within(cover)
x = list(x)
self.assertEqual(len(x), 1)
self.assertEqual(x[0], 1L, "Geom in index doesn't overlap")
x = ix.equals(poly1)
x = list(x)
self.assertEqual(len(x), 1)
self.assertEqual(x[0], 1L, "Geom in index isn't exact")
ix.drop()
def test_insert_multiple(self):
ix = GeoIndex('./test_indices', 'test_insert_multiple', int, int)
ix.bulk_insert([(1, poly1), (2, poly2), (3, poly3)])
x = ix.equals(poly1)
x = list(x)
self.assertEqual(len(x), 1)
self.assertEqual(x[0], 1, "Geom in index for poly 1 isn't exact")
x = ix.equals(poly2)
x = list(x)
self.assertEqual(len(x), 1)
self.assertEqual(x[0], 2, "Geom in index for poly 2 isn't exact")
x = ix.equals(poly3)
x = list(x)
self.assertEqual(len(x), 1)
self.assertEqual(x[0], 3, "Geom in index for poly 3 isn't exact")
x = ix.within(cover)
x = list(x)
self.assertEqual(len(x), 3)
self.assertEqual(sorted(x), [1,2,3], "Didn't pull back all ids")
ix.drop()
class GeoJSONCollection(object, UserDict.DictMixin):
def _normalize_srs(self, fcrs):
# untested, but finished
crs = None
reproject = False # assume we're not going to reproject unless proven otherwise
try: # look for a CRS in the feature or featurecollection
if (isinstance(fcrs, str) or isinstance(fcrs, unicode)) : # if it's a string, we can check to see if it matches the original.
crs = SpatialReference(str(fcrs)) # if it doesn't, set a new one
reproject = True # and reproject
elif isinstance(fcrs, int) and fcrs == self.srid:
reproject = False
elif not isinstance(fcrs, SpatialReference): # if it's not a string, make sure it's a SpatialReference object.
raise ValidationError("feature's 'crs' cannot be interpreted as a SpatialReference by OGR") # otherwise whine.
else:
crs = fcrs
except OGRException as e:
raise ValidationError("feature's 'crs' cannot be interpreted as a SpatialReference by OGR:\n"+ str(e) + "\n" + fcrs)
return reproject, crs
def _get_real_geometry(self, feature):
# untested, but finished
# start by assuming we're just using our own CRS.
fc = feature['type'] == 'FeatureCollection' # if we have a feature collection we have to iterate over all the features
crs, reproject = self._normalize_srs(feature['crs']) if 'crs' in feature else (self.srid, False)
if fc: # if we have a feature collection, collect the geometry
if reproject: # if we have to reproject
for i, f in enumerate(feature['features']): # iterate over all the features
geometry = f['geometry'] if isinstance(f['geometry'], GEOSGeometry) else GEOSGeometry(json.dumps(f['geometry']))
geometry.transform(self.srid) # transform their geometry in place
f['geometry'] = json.loads(geometry.json) # and change the geometry itself in the data
feature['crs'] = self.srid
geometry = (GEOSGeometry(json.dumps(fs['geometry']), srid=crs) for fs in feature['features'])
else: # we have only a single feature
geometry = feature['geometry'] if isinstance(feature['geometry'], GEOSGeometry) else GEOSGeometry(json.dumps(feature['geometry']))
if reproject:
geometry.transform(self.srid) # reproject the feature if necessary
feature['geometry'] = json.loads(geometry.json) # and change the geometry itself in the data.
feature['crs'] = self.srid
return fc, feature, geometry # return a tuple of featurecollection:bool, the original dict, and the GEOS geometry.
def __init__(self, db, collection, index_path=None, srid=None, fc=None, clear=False):
# pull collection metadata from the database if it exists
if clear:
GeoIndex(index_path, collection, _from_objectid, _to_objectid, srid=srid if srid else 4326).drop()
db[collection].drop()
db.geojson__meta.remove(collection)
self.collection_metadata = db.geojson__meta
self.meta = db.geojson__meta.find_one(collection) or {
'srid' : srid,
'index_path' : index_path,
'_id' : collection,
'properties' : {}
}
self.collection_metadata.save(self.meta)
if collection not in db.collection_names():
db.create_collection(collection)
if fc and not self.meta['srid'] and 'crs' in fc:
self.srid = fc['crs'] # however this means crs must conform to an integer srid. This is not preferred by the spec but necessary here.
# instantiate GeoIndex for collection
self.index = GeoIndex(index_path, collection, _from_objectid, _to_objectid, srid=srid if srid else 4326)
self.coll = db[collection]
if fc:
self.insert_features(fc, replace=True)
@property
def srid(self):
return self.meta['srid']
@srid.setter
def srid(self, srid):
self.meta['srid'] = srid
self.collection_metadata.save(self.meta)
def __setitem__(self, key, value):
self.meta['properties'][key] = value
self.collection_metadata.save(self.meta)
def __getitem__(self, key):
return self.meta['properties'][key]
def __delitem__(self, key):
del self.meta['properties'][key]
self.collection_metadata.save(self.meta)
def keys(self):
return self.meta['properties'].keys()
def drop(self):
if hasattr(self, 'index'):
self.index.drop()
if hasattr(self, 'coll'):
self.coll.drop()
if hasattr(self, 'meta'):
self.collection_metadata.remove(self.meta['_id'])
def insert_features(self, fc, replace=False, **kwargs):
if(hasattr(fc, 'keys')):
is_fc, fc, geometry = self._get_real_geometry(fc)
if is_fc:
features = fc['features']
del fc['features']
if 'crs' in fc:
del fc['crs']
fc['crs'] = self.srid
if replace:
self.meta['properties'] = fc
self.collection_metadata.save(self.meta)
else:
for k, v in filter(lambda (x,y): x not in ('crs','type'), fc.items()):
if k not in self:
self[k] = v
elif v != self[k]:
raise KeyError("{k} already in feature collection and is not equal")
fcid = self.meta['_id']
for f in features:
f['_parent'] = fcid
try:
ids = self.coll.insert(features)
except AutoReconnect:
ids = []
for feature in features:
try:
i = self.coll.insert(feature)
ids.append(i)
except AutoReconnect:
log.error('feature {f} wouldnt insert'.format(f=len(ids)))
self.index.bulk_insert(zip(ids, geometry))
else:
oid = self.coll.insert(fc, **kwargs)
self.index.insert(oid, geometry)
else:
def read_bathymetry_file():
log.info('Building irregular-grid index from bathymetry file. This will take awhile.')
then = datetime.now()
def take(k, source):
while k>0:
yield re.split(r'\s*', source.readline().strip())
k -= 1
src = open(settings.BATHYMETRY_SOURCE_FILE)
version = src.readline().strip()
num_edges, num_nodes = [int(k) for k in re.split(r'\s*', src.readline().strip())]
log.info("Reading {num_edges} triangles from {num_nodes} total nodes".format(num_edges=num_edges, num_nodes=num_nodes))
zvalues = np.empty(num_nodes+1, dtype=np.float_)
nodes_arr = np.empty((num_nodes+1, 2), dtype=np.float_)
triangle_arr = np.empty((num_edges+1, 3), dtype=np.int_)
triangle_coords_arr = np.empty((num_edges+1, 3, 2), dtype=np.float_)
#triangle_coords_arr = {}
for node, x, y, z in take(num_nodes, src):
node = int(node)
nodes_arr[node][0] = float(x)
nodes_arr[node][1] = float(y)
zvalues[node] = z
log.info('Built node array')
for triangle, dim, node1, node2, node3 in take(num_edges, src):
node1 = int(node1)
node2 = int(node2)
node3 = int(node3)
triangle = int(triangle)
triangle_arr[triangle][0] = node1
triangle_arr[triangle][1] = node2
triangle_arr[triangle][2] = node3
triangle_coords_arr[triangle][0] = nodes_arr[node1]
triangle_coords_arr[triangle][1] = nodes_arr[node2]
triangle_coords_arr[triangle][2] = nodes_arr[node3]
#triangle_coords_arr[triangle] = Polygon(( tuple(nodes_arr[node1]), tuple(nodes_arr[node2]), tuple(nodes_arr[node3]), tuple(nodes_arr[node1]) ), srid=4326)
log.info('Built triangle array')
obj = StaticArray.objects.filter(name='bathymetry', long_name='bathymetry').first() or StaticArray(name='bathymetry', long_name='bathymetry')
obj.data = pickle.dumps(zvalues)
obj.save()
log.info('Saved bathymetry data')
if os.path.exists(settings.BATHYMETRY_INDEX_FILE + '.idx'):
os.unlink(settings.BATHYMETRY_INDEX_FILE + '.idx')
os.unlink(settings.BATHYMETRY_INDEX_FILE + '.dat')
index = GeoIndex(settings.BATHYMETRY_INDEX_FILE, 'bathymetry', str, int, clear=True) # bulk load
cs = ((i, triangle_arr[i]) for i in range(1, triangle_arr.shape[0]))
triangles = ((i, Polygon(( tuple(nodes_arr[c1]), tuple(nodes_arr[c2]), tuple(nodes_arr[c3]), tuple(nodes_arr[c1]) ), srid=4326) ) for i, (c1, c2, c3) in cs)
index.bulk_insert(triangles)
log.info('saved geometry index')
np.save(settings.BATHYMETRY_INDEX_FILE + "bathymetry.nodes.npy", triangle_arr)
log.info('saved nodes')
np.save(settings.BATHYMETRY_INDEX_FILE + 'bathymetry.coords.npy', triangle_coords_arr)
log.info('saved coordinates')
index.close()
delta = datetime.now() - then
log.info('Finished building index in {secs} seconds.'.format(secs=delta.seconds))
from osgeo import ogr
from ga_spatialnosql.index import GeoIndex
from datetime import datetime
from django.contrib.gis.geos import Polygon
from cera import query
ds3 = GeoIndex('cera/', 'bathymetry',str,int)
query.ensure_bathymetry_index()
then = datetime.now()
x1,y1,x2,y2 = -81,31,-76,36
bbox = Polygon(((x1,y1), (x2,y1), (x2,y2), (x1,y2), (x1,y1)), srid=4326)
l = ds3.bboverlaps(bbox)
fs = list(l)
delta = datetime.now()
print "finished test 1 in {secs}".format(secs=(delta-then).seconds)
then = datetime.now()
g,l = query.bbox_values_for_triangles('maxelev','inundationZeta',datetime(2012,6,28), (-81,31,-76,36))
print l.shape
delta = datetime.now()
print 'finished test 2 in {secs}'.format(secs=(delta-then).seconds)
then = datetime.now()
g, l = query.bbox_mean_values_for_triangles('maxelev','inundationZeta',datetime(2012,6,28), (-81,31,-76,36))
print l.shape
delta = datetime.now()
print 'finished test 3 in {secs}'.format(secs=(delta-then).seconds)
def test_multiple_indexes(self):
ix1 = GeoIndex('./test_indices', 'test_multiple_indexes', int, int)
ix2 = GeoIndex('./test_indices', 'test_multiple_indexes', int, int)
ix1.insert(1L, poly1)
ix2.insert(2L, poly2)
ix1.insert(3L, poly3)
x = ix1.intersects(cover)
x = list(x)
y = ix2.intersects(cover)
y = list(y)
self.assertEqual(len(x), 3, "Didn't pull back all ids with first index")
self.assertEqual(sorted(x), [1,2,3], "Didn't pull back all ids with first index")
self.assertEqual(len(y), 3, "didn't pull back all ids with second index")
self.assertEqual(sorted(y), [1,2,3], "Didn't pull back all ids with second index")
ix1.drop()
ix2.drop()
def test_delete_id(self):
ix = GeoIndex('./test_indices', 'test_delete_id', int, int)
ix.insert(1L, poly1)
ix.insert(2L, poly2)
ix.insert(3L, poly3)
ix.delete(1)
x = ix.within(cover)
x = list(x)
self.assertEqual(len(x), 2, "didn't drop one like we should have")
self.assertEqual(sorted(x), [2,3], "Didn't pull back all ids")
ix.bulk_delete([2,3])
x = ix.within(cover)
x = list(x)
self.assertEqual(len(x) ,0)
ix.drop()
