def _sub(self, data): import datetime if data.get('aa', False): from databundles.geo.analysisarea import get_analysis_area aa = get_analysis_area(self.bundle.library, **data['aa']) aa_d = dict(aa.__dict__) aa_d['aa_name'] = aa_d['name'] del aa_d['name'] data = dict(data.items() + aa_d.items()) data['bundle_name'] = self.bundle.identity.name data['date'] = datetime.datetime.now().date().isoformat() data['query'] = data.get('query','').format(**data) data['extract_where'] = data.get('extract_where','').format(**data) data['title'] = data.get('title','').format(**data) data['description'] = data.get('description','').format(**data) data['name'] = data.get('name','').format(**data) data['layer_name'] = data.get('layer_name','').format(**data) data['path'] = self.bundle.filesystem.path('extracts',format(data['name'])) data['done_if'] = data.get('done_if','').format(**data) return data
def make_hdf(self): import databundles.geo as dg from databundles.geo.analysisarea import get_analysis_area from osgeo.gdalconst import GDT_Float32 import numpy as np place='SndSAN' aa = get_analysis_area(self.library, place=place, scale=10) trans = aa.get_translator() a = aa.new_array() k = dg.GaussianKernel(33,11) k.matrix *= ( 1000000 / aa.scale**2 ) # Convert to count / km^2 p = self.partitions.find(table='pitc10') lr = self.init_log_rate() for row in p.query("""SELECT *, Y(Transform(geometry, 4326)) as lat, X(Transform(geometry, 4326)) as lon FROM pitc10"""): p = trans(row['lon'], row['lat']) k.apply_add(a,p) lr("Add raster point") raster = self.partitions.new_hdf_partition(table='pitc10r') raster.database.put_geo(place, a, aa)
def test_sfschema(self): from databundles.geo.sfschema import TableShapefile from databundles.geo.analysisarea import get_analysis_area _, communities = self.bundle.library.dep('communities') csrs = communities.get_srs() gp = self.bundle.partitions.new_geo_partition(table='geot2') with gp.database.inserter(source_srs=csrs) as ins: for row in communities.query(""" SELECT *, X(Transform(Centroid(geometry), 4326)) AS lon, Y(Transform(Centroid(geometry), 4326)) as lat, AsText(geometry) as wkt, AsBinary(geometry) as wkb FROM communities"""): r = {'name':row['cpname'], 'lat': row['lat'], 'lon': row['lon'], 'wkt': row['wkt']} ins.insert(r) return aa = get_analysis_area(self.bundle.library, geoid = 'CG0666000') path1 = '/tmp/geot1.kml' if os.path.exists(path1): os.remove(path1) sfs1 = TableShapefile(self.bundle, path1, 'geot1' ) path2 = '/tmp/geot2.kml' if os.path.exists(path2): os.remove(path2) sfs2 = TableShapefile(self.bundle, path2, 'geot2', source_srs=communities.get_srs()) print sfs1.type, sfs2.type for row in communities.query(""" SELECT *, X(Transform(Centroid(geometry), 4326)) AS lon, Y(Transform(Centroid(geometry), 4326)) as lat, AsText(geometry) as wkt, AsBinary(geometry) as wkb FROM communities"""): sfs1.add_feature( {'name':row['cpname'], 'lat': row['lat'], 'lon': row['lon'], 'wkt': row['wkt']}) sfs2.add_feature( {'name':row['cpname'], 'lat': row['lat'], 'lon': row['lon'], 'wkt': row['wkt']}) sfs1.close() sfs2.close()
def x_test_basic(self): from databundles.geo.analysisarea import get_analysis_area, draw_edges from databundles.geo import Point from databundles.geo.kernel import GaussianKernel aa = get_analysis_area(self.bundle.library, geoid = 'CG0666000') a = aa.new_array() #draw_edges(a) print a.shape, a.size gaussian = GaussianKernel(11,6) for i in range(0,400, 20): p = Point(100+i,100+i) gaussian.apply_add(a,p) aa.write_geotiff('/tmp/box.tiff', a, data_type=GDT_Float32)
def demo(self): '''A commented demonstration of how to create crime data extracts as GeoTIFF images Run with: python bundle.py run demo ''' from databundles.geo.analysisarea import get_analysis_area, draw_edges from databundles.geo.util import create_bb from databundles.geo import Point from databundles.geo.kernel import GaussianKernel from databundles.geo.array import statistics, unity_norm, std_norm from osgeo.gdalconst import GDT_Float32, GDT_Byte, GDT_Int16 from numpy import ma import random # Get the San Diego analysis area from the GEOID ( Defined by the US Census) # you can look up geoids in clarinova.com-extents-2012-7ba4/meta/san-diego-places.csv, # or query the places table in clarinova.com-extents-2012-7ba4.db aa = get_analysis_area(self.library, geoid = 'CG0666000') # Get a function to translate coodinates from the default lat/lon, WGS84, # into the cordinate system of the AnalysisArea, which in this case # is 20m square cells in an area based on a California StatePlane Zone trans = aa.get_translator() print "\n---- Display Analysis Area ----" print aa # This should print a small value, something close to (0,0). # It won't be exactly (0,0), since the analysis area envelope must be # larger than the envelop of the place to account for rotation from # re-projection print "Origin", trans(aa.lonmin, aa.latmin) # At the Sandiego latitude, 1/5000 of a degree, .0002, is about 20 meters, # So incrementing by that amount should advance our cell position by one print "\n---- Check translation function ----" import numpy as np for i,x in enumerate(np.arange(0,.002,.0002)): print i,x,trans(aa.lonmin+x, aa.latmin+x) # Now we can load in the crime incident data, translate the lat/lon points # to our array coordinates, and produce an image. # Get a reference to the bundle named as "crime" in the bundle.yaml configuration # file. crime = spotcrime.com-us_crime_incidents-orig-7ba4 r = self.library.dep('crime') # Fill in the values for the extents of the analysis area into the # query template. q = self.config.build.incident_query.format(**aa.__dict__) q += " AND type = 'Theft' " # A 'Kernel' is a matrix in a process called 'convolution'. We're doing something # somewhat different, but are re-using the name. This kernel is added # onto the output array for each crime incident, and represents a Normal # distribution, so it spreads out the influence over a larger area than # a single cell. # The matrix is square, 9 cells to a side. The function has 1/2 of its # maximun ( Full-Width-Half Maximum, FWHM) three cells from the center. kernel = GaussianKernel(33,11) # We're going to need an output array. This creates a numpy array that # has the correct size a = aa.new_array() # Main array ar = aa.new_array() # Array with random perturbation rs = 4 print "Array shape: ",a.shape for i,row in enumerate(r.bundle.database.connection.execute(q)): if i > 0 and i%1000 == 0: print "Processed {} rows".format(i) if i > 5000: break point = trans(row['longitude'], row['latitude']) kernel.apply_add(a,point) # The source data is coded to the 'hundred block' address, # such as: 12XX Main Street. This make the points quantized, so # add a little randomness for a smoother map. rpoint = Point(point.x+random.randint(-rs, rs), point.y+random.randint(-rs, rs)) kernel.apply_add(ar,rpoint) # make a helper to store files in the extracts directory ed = lambda f: self.filesystem.path('extracts','demo',f+'.tiff') print "\n--- Statistics, Before Normalizing ---" print statistics(a) aa.write_geotiff(ed('orig'), a, type_=GDT_Float32) print "\n--- Statistics, After Masking Normalizing ---" # # Masking marks some values as invalid, so they don't get used in statistics. # I this case, we are making 0 invalid, which will keep it from being # considered in the std deviation later in std_norm. a = ma.masked_equal(a,0) print statistics(a) aa.write_geotiff(ed('masked'), a, type_=GDT_Float32) print "\n--- Statistics, After StdDev Normalizing ---" o = std_norm(a) print statistics(o) aa.write_geotiff(ed('stddev'), o, type_=GDT_Float32) print "\n--- Statistics, After Unity Normalizing ---" o = unity_norm(a) print statistics(o) aa.write_geotiff(ed('unity'), o, type_=GDT_Float32) # Write the array with randomness ar = ma.masked_equal(ar,0) aa.write_geotiff('/tmp/random.tiff', std_norm(ar), type_=GDT_Float32)