def create_datasource(self, path, fmt):
import os
from databundles.util import rm_rf
options = []
if fmt == 'kml':
drv = ogr.GetDriverByName( "KML" )
elif fmt == 'geojson':
drv = ogr.GetDriverByName( "GeoJSON" )
elif fmt == 'sqlite' or fmt == 'db':
drv = ogr.GetDriverByName( "SQLite" )
options = ['SPATIALITE=YES', 'INIT_WITH_EPSG=YES','OGR_SQLITE_SYNCHRONOUS=OFF',
'OGR_SQLITE_CACHE=1024', '-gt 50000', 'COMPRESS_GEOM=yes']
elif fmt == 'shapefile':
drv = ogr.GetDriverByName( "ESRI Shapefile" )
else:
raise Exception("Unknown format: {} ".format(fmt))
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
if os.path.exists(path):
if os.path.isdir(path):
rm_rf(path)
else:
os.remove(path)
ds = drv.CreateDataSource(path, options=options)
if ds is None:
raise Exception("Failed to create datasource: {}".format(path))
return ds
def deinit(self):
import os
fn = os.path.join(self.dir_, '.gitignore')
if os.path.exists(fn):
os.remove(fn)
dn = os.path.join(self.dir_, '.git')
if os.path.exists(dn):
from databundles.util import rm_rf
rm_rf(dn)
def setUp(self):
rm_rf('/tmp/server')
self.copy_or_build_bundle()
self.bundle_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),'testbundle')
self.rc = RunConfig([os.path.join(self.bundle_dir,'client-test-config.yaml'),
os.path.join(self.bundle_dir,'bundle.yaml')])
self.server_rc = RunConfig([os.path.join(self.bundle_dir,'server-test-config.yaml')])
self.bundle = Bundle()
self.bundle_dir = self.bundle.bundle_dir
def _test_put_bundle(self, name, remote_config=None):
from databundles.bundle import DbBundle
from databundles.library import QueryCommand
rm_rf('/tmp/server')
self.start_server(remote_config)
r = None #Rest(self.server_url, remote_config)
bf = self.bundle.database.path
# With an FLO
response = r.put(open(bf), self.bundle.identity)
self.assertEquals(self.bundle.identity.id_, response.object.get('id'))
# with a path
response = r.put( bf, self.bundle.identity)
self.assertEquals(self.bundle.identity.id_, response.object.get('id'))
for p in self.bundle.partitions.all:
response = r.put( open(p.database.path), p.identity)
self.assertEquals(p.identity.id_, response.object.get('id'))
# Now get the bundles
bundle_file = r.get(self.bundle.identity,'/tmp/foo.db')
bundle = DbBundle(bundle_file)
self.assertIsNot(bundle, None)
self.assertEquals('a1DxuZ',bundle.identity.id_)
# Should show up in datasets list.
o = r.list()
self.assertTrue('a1DxuZ' in o.keys() )
o = r.find(QueryCommand().table(name='tone').partition(any=True))
self.assertTrue( 'b1DxuZ001' in [i.id_ for i in o])
self.assertTrue( 'a1DxuZ' in [i.as_dataset.id_ for i in o])
def test_caches(self):
from functools import partial
fn = '/tmp/1mbfile'
get_cache = partial(self.bundle.filesystem._get_cache, self.rc.filesystem)
with open(fn, 'wb') as f:
f.write('.'*(1024))
#get_cache('cache1').put(fn,'cache1')
#get_cache('cache2').put(fn,'cache2')
get_cache('cache3').put(fn,'cache3')
rm_rf(get_cache('cache3').cache_dir)
path = get_cache('cache3').get('cache3')
self.assertEquals('AKIAIOKK4KSYYGYXWQJQ', get_cache('cache3').connection_info.get('access_key'))
def bound_clusters_in_raster( a, aa, shape_file_dir,
contour_interval,contour_value, use_bb=True, use_distance=False):
"""Create a shapefile that contains contours and bounding boxes for clusters
of contours.
:param a: A numpy array that contains the data inwhich to find clusters
:type a: Numpy array
:param aa: The analysis object that sets the coordinate system for the area that contains the array
:type aa: databundles.geo.AnalysisArea
:param shape_file_dir: The path to a directory where generated files will be stored.
:type shape_file_dir: string
:param contour_interval: The difference between successive contour intervals.
:type contour_interval: float
:param contour_value:
:type contour_value: float
:param use_bb: If True, compute nearness and intersection using the contours bounding boxes, not the geometry
:type use_bb: bool
:param use_distance: If not False, consider contours that are closer than this value to be overlapping.
:type : number
:rtype: Returns a list of dictionaries, one for each of the combined bounding boxes
This method will store, in the `shape_file_dir` directory:
* a GeoTIFF representation of the array `a`
* An ERSI shapefile layer named `countours`, holding all of the countours.
* A layer named `contour_bounds` with the bounding boxes for all of the contours with value `contour_value`
* A layer named `combined_bounds` with bounding boxes of intersecting and nearby boxes rom `contour_bounds`
The routine will iteratively combine contours that overlap.
If `use_distance` is set to a number, and contours that are closer than this value will be joined.
If `use_bb` is set, the intersection and distance computations use the bounding boxes of the contours,
not the contours themselves.
"""
import databundles.geo as dg
from osgeo.gdalconst import GDT_Float32
import databundles.util as util
from osgeo import gdal
import ogr, os
import numpy as np
if os.path.exists(shape_file_dir):
util.rm_rf(shape_file_dir)
os.makedirs(shape_file_dir)
rasterf = os.path.join(shape_file_dir,'contour.tiff')
ogr_ds = ogr.GetDriverByName('ESRI Shapefile').CreateDataSource(shape_file_dir)
# Setup the countour layer.
ogr_lyr = ogr_ds.CreateLayer('contours', aa.srs)
ogr_lyr.CreateField(ogr.FieldDefn('id', ogr.OFTInteger))
ogr_lyr.CreateField(ogr.FieldDefn('value', ogr.OFTReal))
# Create the contours from the GeoTIFF file.
ds = aa.get_geotiff(rasterf, data_type=GDT_Float32)
ds.GetRasterBand(1).SetNoDataValue(0)
ds.GetRasterBand(1).WriteArray(np.flipud(a))
gdal.ContourGenerate(ds.GetRasterBand(1),
contour_interval, # contourInterval
0, # contourBase
[], # fixedLevelCount
0, # useNoData
0, # noDataValue
ogr_lyr, #destination layer
0, #idField
1 # elevation field
)
# Get buffered bounding boxes around each of the hotspots,
# and put them into a new layer.
bound_lyr = ogr_ds.CreateLayer('contour_bounds', aa.srs)
for i in range(ogr_lyr.GetFeatureCount()):
f1 = ogr_lyr.GetFeature(i)
if f1.GetFieldAsDouble('value') != contour_value:
continue
g1 = f1.GetGeometryRef()
bb = dg.create_bb(g1.GetEnvelope(), g1.GetSpatialReference())
f = ogr.Feature(bound_lyr.GetLayerDefn())
f.SetGeometry(bb)
bound_lyr.CreateFeature(f)
# Doing a full loop instead of a list comprehension b/c the way that comprehensions
# compose arrays results in segfaults, probably because a copied geometry
# object is being released before being used.
geos = []
for i in range(bound_lyr.GetFeatureCount()):
f = bound_lyr.GetFeature(i)
g = f.geometry()
geos.append(g.Clone())
# Combine hot spots that have intersecting bounding boxes, to get larger
# areas that cover all of the adjacent intersecting smaller areas.
geos = dg.combine_envelopes(geos, use_bb=use_bb, use_distance = use_distance)
# Write out the combined bounds areas.
lyr = ogr_ds.CreateLayer('combined_bounds', aa.srs)
lyr.CreateField(ogr.FieldDefn('id', ogr.OFTInteger))
lyr.CreateField(ogr.FieldDefn('area', ogr.OFTReal))
lyr.CreateField(ogr.FieldDefn('name', ogr.OFTString))
lyr.CreateField(ogr.FieldDefn('code', ogr.OFTString))
envelopes = []
id = 1
for env in geos:
f = ogr.Feature(lyr.GetLayerDefn())
bb = dg.create_bb(env.GetEnvelope(), env.GetSpatialReference())
f.SetGeometry(bb)
f.SetField(0, id)
f.SetField(1, bb.Area())
f.SetField(2, None)
f.SetField(3, None)
id += 1
lyr.CreateFeature(f)
envelopes.append({'id':id, 'env':bb.GetEnvelope(), 'area':bb.Area()})
return envelopes
def x_test_put_redirect(self):
from databundles.bundle import DbBundle
from databundles.library import QueryCommand
from databundles.util import md5_for_file, rm_rf, bundle_file_type
#
# Simple out and retrieve
#
cache = self.bundle.filesystem._get_cache(self.server_rc.filesystem, 'direct-remote')
cache2 = self.bundle.filesystem._get_cache(self.server_rc.filesystem, 'direct-remote-2')
rm_rf(os.path.dirname(cache.cache_dir))
rm_rf(os.path.dirname(cache2.cache_dir))
cache.put( self.bundle.database.path, 'direct')
path = cache2.get('direct')
self.assertEquals('sqlite',bundle_file_type(path))
cache.remove('direct', propagate = True)
#
# Connect through server.
#
rm_rf('/tmp/server')
self.start_server(name='default-remote')
api = None # Rest(self.server_url, self.rc.accounts)
# Upload directly, then download via the cache.
cache.remove(self.bundle.identity.cache_key, propagate = True)
r = api.upload_file(self.bundle.identity, self.bundle.database.path, force=True )
path = cache.get(self.bundle.identity.cache_key)
b = DbBundle(path)
self.assertEquals("source-dataset-subset-variation-ca0d",b.identity.name )
#
# Full service
#
p = self.bundle.partitions.all[0]
cache.remove(self.bundle.identity.cache_key, propagate = True)
cache.remove(p.identity.cache_key, propagate = True)
r = api.put( self.bundle.database.path, self.bundle.identity )
print "Put {}".format(r.object)
r = api.put(p.database.path, p.identity )
print "Put {}".format(r.object)
r = api.put(p.database.path, p.identity )
r = api.get(p.identity,'/tmp/foo.db')
print "Get {}".format(r)
b = DbBundle(r)
self.assertEquals("source-dataset-subset-variation-ca0d",b.identity.name )
