def _sub(self, data):
import datetime
if data.get('aa', False):
from databundles.geo.analysisarea import get_analysis_area
aa = get_analysis_area(self.bundle.library, **data['aa'])
aa_d = dict(aa.__dict__)
aa_d['aa_name'] = aa_d['name']
del aa_d['name']
data = dict(data.items() + aa_d.items())
data['bundle_name'] = self.bundle.identity.name
data['date'] = datetime.datetime.now().date().isoformat()
data['query'] = data.get('query','').format(**data)
data['extract_where'] = data.get('extract_where','').format(**data)
data['title'] = data.get('title','').format(**data)
data['description'] = data.get('description','').format(**data)
data['name'] = data.get('name','').format(**data)
data['layer_name'] = data.get('layer_name','').format(**data)
data['path'] = self.bundle.filesystem.path('extracts',format(data['name']))
data['done_if'] = data.get('done_if','').format(**data)
return data
def make_hdf(self):
import databundles.geo as dg
from databundles.geo.analysisarea import get_analysis_area
from osgeo.gdalconst import GDT_Float32
import numpy as np
place='SndSAN'
aa = get_analysis_area(self.library, place=place, scale=10)
trans = aa.get_translator()
a = aa.new_array()
k = dg.GaussianKernel(33,11)
k.matrix *= ( 1000000 / aa.scale**2 ) # Convert to count / km^2
p = self.partitions.find(table='pitc10')
lr = self.init_log_rate()
for row in p.query("""SELECT *,
Y(Transform(geometry, 4326)) as lat, X(Transform(geometry, 4326)) as lon
FROM pitc10"""):
p = trans(row['lon'], row['lat'])
k.apply_add(a,p)
lr("Add raster point")
raster = self.partitions.new_hdf_partition(table='pitc10r')
raster.database.put_geo(place, a, aa)
def test_sfschema(self):
from databundles.geo.sfschema import TableShapefile
from databundles.geo.analysisarea import get_analysis_area
_, communities = self.bundle.library.dep('communities')
csrs = communities.get_srs()
gp = self.bundle.partitions.new_geo_partition(table='geot2')
with gp.database.inserter(source_srs=csrs) as ins:
for row in communities.query("""
SELECT *,
X(Transform(Centroid(geometry), 4326)) AS lon,
Y(Transform(Centroid(geometry), 4326)) as lat,
AsText(geometry) as wkt,
AsBinary(geometry) as wkb
FROM communities"""):
r = {'name':row['cpname'], 'lat': row['lat'], 'lon': row['lon'], 'wkt': row['wkt']}
ins.insert(r)
return
aa = get_analysis_area(self.bundle.library, geoid = 'CG0666000')
path1 = '/tmp/geot1.kml'
if os.path.exists(path1): os.remove(path1)
sfs1 = TableShapefile(self.bundle, path1, 'geot1' )
path2 = '/tmp/geot2.kml'
if os.path.exists(path2): os.remove(path2)
sfs2 = TableShapefile(self.bundle, path2, 'geot2', source_srs=communities.get_srs())
print sfs1.type, sfs2.type
for row in communities.query("""
SELECT *,
X(Transform(Centroid(geometry), 4326)) AS lon,
Y(Transform(Centroid(geometry), 4326)) as lat,
AsText(geometry) as wkt,
AsBinary(geometry) as wkb
FROM communities"""):
sfs1.add_feature( {'name':row['cpname'], 'lat': row['lat'], 'lon': row['lon'], 'wkt': row['wkt']})
sfs2.add_feature( {'name':row['cpname'], 'lat': row['lat'], 'lon': row['lon'], 'wkt': row['wkt']})
sfs1.close()
sfs2.close()
def x_test_basic(self):
from databundles.geo.analysisarea import get_analysis_area, draw_edges
from databundles.geo import Point
from databundles.geo.kernel import GaussianKernel
aa = get_analysis_area(self.bundle.library, geoid = 'CG0666000')
a = aa.new_array()
#draw_edges(a)
print a.shape, a.size
gaussian = GaussianKernel(11,6)
for i in range(0,400, 20):
p = Point(100+i,100+i)
gaussian.apply_add(a,p)
aa.write_geotiff('/tmp/box.tiff', a, data_type=GDT_Float32)
def demo(self):
'''A commented demonstration of how to create crime data extracts as GeoTIFF
images
Run with: python bundle.py run demo
'''
from databundles.geo.analysisarea import get_analysis_area, draw_edges
from databundles.geo.util import create_bb
from databundles.geo import Point
from databundles.geo.kernel import GaussianKernel
from databundles.geo.array import statistics, unity_norm, std_norm
from osgeo.gdalconst import GDT_Float32, GDT_Byte, GDT_Int16
from numpy import ma
import random
# Get the San Diego analysis area from the GEOID ( Defined by the US Census)
# you can look up geoids in clarinova.com-extents-2012-7ba4/meta/san-diego-places.csv,
# or query the places table in clarinova.com-extents-2012-7ba4.db
aa = get_analysis_area(self.library, geoid = 'CG0666000')
# Get a function to translate coodinates from the default lat/lon, WGS84,
# into the cordinate system of the AnalysisArea, which in this case
# is 20m square cells in an area based on a California StatePlane Zone
trans = aa.get_translator()
print "\n---- Display Analysis Area ----"
print aa
# This should print a small value, something close to (0,0).
# It won't be exactly (0,0), since the analysis area envelope must be
# larger than the envelop of the place to account for rotation from
# re-projection
print "Origin", trans(aa.lonmin, aa.latmin)
# At the Sandiego latitude, 1/5000 of a degree, .0002, is about 20 meters,
# So incrementing by that amount should advance our cell position by one
print "\n---- Check translation function ----"
import numpy as np
for i,x in enumerate(np.arange(0,.002,.0002)):
print i,x,trans(aa.lonmin+x, aa.latmin+x)
# Now we can load in the crime incident data, translate the lat/lon points
# to our array coordinates, and produce an image.
# Get a reference to the bundle named as "crime" in the bundle.yaml configuration
# file. crime = spotcrime.com-us_crime_incidents-orig-7ba4
r = self.library.dep('crime')
# Fill in the values for the extents of the analysis area into the
# query template.
q = self.config.build.incident_query.format(**aa.__dict__)
q += " AND type = 'Theft' "
# A 'Kernel' is a matrix in a process called 'convolution'. We're doing something
# somewhat different, but are re-using the name. This kernel is added
# onto the output array for each crime incident, and represents a Normal
# distribution, so it spreads out the influence over a larger area than
# a single cell.
# The matrix is square, 9 cells to a side. The function has 1/2 of its
# maximun ( Full-Width-Half Maximum, FWHM) three cells from the center.
kernel = GaussianKernel(33,11)
# We're going to need an output array. This creates a numpy array that
# has the correct size
a = aa.new_array() # Main array
ar = aa.new_array() # Array with random perturbation
rs = 4
print "Array shape: ",a.shape
for i,row in enumerate(r.bundle.database.connection.execute(q)):
if i > 0 and i%1000 == 0:
print "Processed {} rows".format(i)
if i > 5000:
break
point = trans(row['longitude'], row['latitude'])
kernel.apply_add(a,point)
# The source data is coded to the 'hundred block' address,
# such as: 12XX Main Street. This make the points quantized, so
# add a little randomness for a smoother map.
rpoint = Point(point.x+random.randint(-rs, rs),
point.y+random.randint(-rs, rs))
kernel.apply_add(ar,rpoint)
# make a helper to store files in the extracts directory
ed = lambda f: self.filesystem.path('extracts','demo',f+'.tiff')
print "\n--- Statistics, Before Normalizing ---"
print statistics(a)
aa.write_geotiff(ed('orig'), a, type_=GDT_Float32)
print "\n--- Statistics, After Masking Normalizing ---"
#
# Masking marks some values as invalid, so they don't get used in statistics.
# I this case, we are making 0 invalid, which will keep it from being
# considered in the std deviation later in std_norm.
a = ma.masked_equal(a,0)
print statistics(a)
aa.write_geotiff(ed('masked'), a, type_=GDT_Float32)
print "\n--- Statistics, After StdDev Normalizing ---"
o = std_norm(a)
print statistics(o)
aa.write_geotiff(ed('stddev'), o, type_=GDT_Float32)
print "\n--- Statistics, After Unity Normalizing ---"
o = unity_norm(a)
print statistics(o)
aa.write_geotiff(ed('unity'), o, type_=GDT_Float32)
# Write the array with randomness
ar = ma.masked_equal(ar,0)
aa.write_geotiff('/tmp/random.tiff', std_norm(ar), type_=GDT_Float32)