def handle(self, *args, **options):
# Load configuration
sys.path.append(options['data_dir'])
from definitions import SHAPEFILES
if options['only']:
only = options['only'].upper().split(',')
# TODO: stripping whitespace here because optparse doesn't handle
# it correctly
sources = [
s for s in SHAPEFILES if s.replace(' ', '').upper() in only
]
elif options['except']:
exceptions = options['except'].upper().split(',')
# See above
sources = [
s for s in SHAPEFILES
if s.replace(' ', '').upper() not in exceptions
]
else:
sources = [s for s in SHAPEFILES]
for kind, config in SHAPEFILES.items():
if kind not in sources:
log.info('Skipping %s.' % kind)
continue
log.info('Processing %s.' % kind)
self.load_set(kind, config, options)
def handle(self, *args, **options):
# Load configuration
sys.path.append(options['data_dir'])
from definitions import SHAPEFILES
if options['only']:
only = options['only'].upper().split(',')
# TODO: stripping whitespace here because optparse doesn't handle
# it correctly
sources = [s for s in SHAPEFILES
if s.replace(' ', '').upper() in only]
elif options['except']:
exceptions = options['except'].upper().split(',')
# See above
sources = [s for s in SHAPEFILES
if s.replace(' ', '').upper() not in exceptions]
else:
sources = [s for s in SHAPEFILES]
for kind, config in SHAPEFILES.items():
if kind not in sources:
log.info('Skipping %s.' % kind)
continue
log.info('Processing %s.' % kind)
self.load_set(kind, config, options)
def handle(self, *args, **options):
sys.path.append(os.path.join(settings.SITE_DIR, SHAPEFILES_DIR))
from definitions import SHAPEFILES
for name, options in SHAPEFILES.items():
self.stdout.write('Processing \'{}\' now...'.format(name))
self.stdout.write('Attempting to create collection...')
collection, _ = Collection.objects.get_or_create(
name=name,
authority=options['authority'],
last_updated=options['last_updated'],
count=0,
slug=options['slug'],
source_url=options['source_url']
)
if _ is True:
self.stdout.write('\'{}\' collection created!'.format(name))
else:
self.stdout.write('\'{}\' collection already exists!'
' Not a problem.'.format(name))
lm = LayerMapping(Shape,
os.path.join(SHAPEFILES_DIR, options['file']),
options['layer_mapping'],
encoding='latin-1')
lm.save(verbose=True, strict=True)
def handle(self, *args, **options):
sys.path.append(os.path.join(settings.SITE_DIR, SHAPEFILES_DIR))
from definitions import SHAPEFILES
for name, options in SHAPEFILES.items():
self.stdout.write('Processing \'{}\' now...'.format(name))
self.stdout.write('Attempting to create collection...')
collection, _ = Collection.objects.get_or_create(
name=name,
authority=options['authority'],
last_updated=options['last_updated'],
count=0,
slug=options['slug'],
source_url=options['source_url'])
if _ is True:
self.stdout.write('\'{}\' collection created!'.format(name))
else:
self.stdout.write('\'{}\' collection already exists!'
' Not a problem.'.format(name))
lm = LayerMapping(Shape,
os.path.join(SHAPEFILES_DIR, options['file']),
options['layer_mapping'],
encoding='latin-1')
lm.save(verbose=True, strict=True)
def handle(self, *args, **options):
# Load configuration
sys.path.append(options['data_dir'])
from definitions import SHAPEFILES
if options['only']:
only = options['only'].split(',')
# TODO: stripping whitespace here because optparse doesn't handle it correctly
sources = [s for s in SHAPEFILES if s.replace(' ', '') in only]
elif options['except']:
exceptions = options['except'].upper().split(',')
# See above
sources = [s for s in SHAPEFILES if s.replace(' ', '') not in exceptions]
else:
sources = [s for s in SHAPEFILES]
for kind, config in SHAPEFILES.items():
if kind not in sources:
log.info('Skipping %s.' % kind)
continue
log.info('Processing %s.' % kind)
if options['clear']:
set = None
try:
set = BoundarySet.objects.get(name=kind)
if set:
log.info('Clearing old %s.' % kind)
set.boundaries.all().delete()
set.delete()
log.info('Loading new %s.' % kind)
except BoundarySet.DoesNotExist:
log.info("No existing boundary set of kind [%s] so nothing to delete" % kind)
path = os.path.join(options['data_dir'], config['file'])
datasources = create_datasources(path)
layer = datasources[0][0]
# Create BoundarySet
try:
set = BoundarySet.objects.get(name=kind)
log.info("Using existing BoundarySet [%s]" % set.slug)
except:
set = BoundarySet.objects.create(
name=kind,
singular=config['singular'],
kind_first=config['kind_first'],
authority=config['authority'],
domain=config['domain'],
last_updated=config['last_updated'],
href=config['href'],
notes=config['notes'],
count=len(layer),
metadata_fields=layer.fields)
log.info("Created new BoundarySet [%s]" % set.slug)
for datasource in datasources:
log.info("Loading %s from %s" % (kind, datasource.name))
# Assume only a single-layer in shapefile
if datasource.layer_count > 1:
log.warn('%s shapefile [%s] has multiple layers, using first.' % (datasource.name, kind))
layer = datasource[0]
self.add_boundaries_for_layer(config,layer,set)
set.count = Boundary.objects.filter(set=set).count() # sync this with reality
set.save()
# TODO: work through additional shapefiles, increment set.count
log.info('%s count: %i' % (kind, set.count))
#!/usr/bin/env python
import sys
from os.path import dirname, join, splitext
from subprocess import call
# add the shapefiles dir to the path
shapefiles_path = join(dirname(__file__), 'shapefiles')
sys.path.append(shapefiles_path)
# load our shapefile definitions
from definitions import SHAPEFILES
if 'geojson' in sys.argv:
for title, data in SHAPEFILES.items():
shapefile = join('shapefiles', data['file'])
geojsonfile = splitext(shapefile)[0] + '.geojson'
proc = call(['ogr2ogr', '-f', 'geoJSON', geojsonfile, shapefile])
print "* %s [source](%s) [map](%s)" % (
title, data["href"], geojsonfile)
def handle(self, *args, **options):
# Load configuration
sys.path.append(os.path.abspath(SHAPEFILES_DIR))
from definitions import SHAPEFILES
if options['only']:
only = options['only'].split(',')
# TODO: stripping whitespace here because optparse doesn't handle it correclty
sources = [s for s in SHAPEFILES if s.replace(' ', '') in only]
elif options['except']:
exceptions = options['except'].upper().split(',')
# See above
sources = [s for s in SHAPEFILES if s.replace(' ', '') not in exceptions]
else:
sources = [s for s in SHAPEFILES]
# Get spatial reference system for the postgis geometry field
geometry_field = Boundary._meta.get_field_by_name(GEOMETRY_COLUMN)[0]
SpatialRefSys = connections[DEFAULT_DB_ALIAS].ops.spatial_ref_sys()
db_srs = SpatialRefSys.objects.get(srid=geometry_field.srid).srs
for kind, config in SHAPEFILES.items():
if kind not in sources:
log.info('Skipping %s.' % kind)
continue
log.info('Processing %s.' % kind)
if options['clear']:
set = None
try:
set = BoundarySet.objects.get(name=kind)
except BoundarySet.DoesNotExist:
pass
if set:
log.info('Clearing old %s.' % kind)
set.boundaries.all().delete()
set.delete()
log.info('Loading new %s.' % kind)
path = os.path.join(SHAPEFILES_DIR, config['file'])
datasource = DataSource(path)
# Assume only a single-layer in shapefile
if datasource.layer_count > 1:
log.warn('%s shapefile has multiple layers, using first.' % kind)
layer = datasource[0]
if 'srid' in config:
layer_srs = SpatialRefSys.objects.get(srid=config['srid']).srs
else:
layer_srs = layer.srs
# Create a convertor to turn the source data into
transformer = CoordTransform(layer_srs, db_srs)
# Create BoundarySet
set = BoundarySet.objects.create(
name=kind,
singular=config['singular'],
kind_first=config['kind_first'],
authority=config['authority'],
domain=config['domain'],
last_updated=config['last_updated'],
href=config['href'],
notes=config['notes'],
count=len(layer),
metadata_fields=layer.fields)
for feature in layer:
# Transform the geometry to the correct SRS
geometry = self.polygon_to_multipolygon(feature.geom)
geometry.transform(transformer)
# Create simplified geometry field by collapsing points within 1/1000th of a degree.
# Since Chicago is at approx. 42 degrees latitude this works out to an margin of
# roughly 80 meters latitude and 112 meters longitude.
# Preserve topology prevents a shape from ever crossing over itself.
simple_geometry = geometry.geos.simplify(0.0001, preserve_topology=True)
# Conversion may force multipolygons back to being polygons
simple_geometry = self.polygon_to_multipolygon(simple_geometry.ogr)
# Extract metadata into a dictionary
metadata = {}
for field in layer.fields:
# Decode string fields using encoding specified in definitions config
if config['encoding'] != '':
try:
metadata[field] = feature.get(field).decode(config['encoding'])
# Only strings will be decoded, get value in normal way if int etc.
except AttributeError:
metadata[field] = feature.get(field)
else:
metadata[field] = feature.get(field)
external_id = config['ider'](feature)
feature_name = config['namer'](feature)
# If encoding is specified, decode id and feature name
if config['encoding'] != '':
external_id = external_id.decode(config['encoding'])
feature_name = feature_name.decode(config['encoding'])
if config['kind_first']:
display_name = '%s %s' % (config['singular'], feature_name)
else:
display_name = '%s %s' % (feature_name, config['singular'])
Boundary.objects.create(
set=set,
kind=config['singular'],
external_id=external_id,
name=feature_name,
display_name=display_name,
metadata=metadata,
shape=geometry.wkt,
simple_shape=simple_geometry.wkt,
centroid=geometry.geos.centroid)
log.info('Saved %i %s.' % (set.count, kind))
def handle(self, *args, **options):
# Load configuration
sys.path.append(os.path.abspath(SHAPEFILES_DIR))
from definitions import SHAPEFILES
if options['only']:
only = options['only'].split(',')
# TODO: stripping whitespace here because optparse doesn't handle
# it correclty
sources = [s for s in SHAPEFILES if s.replace(' ', '') in only]
elif options['except']:
exceptions = options['except'].upper().split(',')
# See above
sources = [s for s in SHAPEFILES if s.replace(' ', '') not in exceptions]
else:
sources = [s for s in SHAPEFILES]
# Get spatial reference system for the postgis geometry field
geometry_field = Boundary._meta.get_field_by_name(GEOMETRY_COLUMN)[0]
SpatialRefSys = connections[DEFAULT_DB_ALIAS].ops.spatial_ref_sys()
db_srs = SpatialRefSys.objects.get(srid=geometry_field.srid).srs
for kind, config in SHAPEFILES.items():
if kind not in sources:
log.info('Skipping %s.' % kind)
continue
log.info('Processing %s.' % kind)
if options['clear']:
set = None
try:
set = BoundarySet.objects.get(name=kind)
except BoundarySet.DoesNotExist:
pass
if set:
log.info('Clearing old %s.' % kind)
set.boundaries.all().delete()
set.delete()
log.info('Loading new %s.' % kind)
path = os.path.join(SHAPEFILES_DIR, config['file'])
datasource = DataSource(path)
# Assume only a single-layer in shapefile
if datasource.layer_count > 1:
log.warn('%s shapefile has multiple layers, using first.' % kind)
layer = datasource[0]
if 'srid' in config and config['srid']:
layer_srs = SpatialRefSys.objects.get(srid=config['srid']).srs
else:
layer_srs = layer.srs
# Create a convertor to turn the source data into
transformer = CoordTransform(layer_srs, db_srs)
# Create BoundarySet
set = BoundarySet.objects.create(
name=kind,
singular=config['singular'],
kind_first=config['kind_first'],
authority=config['authority'],
domain=config['domain'],
last_updated=config['last_updated'],
href=config['href'],
notes=config['notes'],
count=len(layer),
metadata_fields=layer.fields)
for feature in layer:
# Transform the geometry to the correct SRS
geometry = self.polygon_to_multipolygon(feature.geom)
geometry.transform(transformer)
# Create simplified geometry field by collapsing points
# within 1/1000th of a degree. Since Chicago is at approx.
# 42 degrees latitude this works out to an margin of roughly
# 80 meters latitude and 112 meters longitude. Preserve
# topology prevents a shape from ever crossing over itself.
simple_geometry = geometry.geos.simplify(0.0001, preserve_topology=True)
# Conversion may force multipolygons back to being polygons
simple_geometry = self.polygon_to_multipolygon(simple_geometry.ogr)
# Extract metadata into a dictionary
metadata = {}
for field in layer.fields:
# Decode string fields using encoding specified in
# definitions config
if config['encoding'] != '':
try:
metadata[field] = feature.get(field).decode(config['encoding'])
# Only strings will be decoded, get value in normal way if int etc.
except AttributeError:
metadata[field] = feature.get(field)
else:
metadata[field] = feature.get(field)
external_id = config['ider'](feature)
feature_name = config['namer'](feature)
# If encoding is specified, decode id and feature name
if config['encoding'] != '':
external_id = external_id.decode(config['encoding'])
feature_name = feature_name.decode(config['encoding'])
if config['kind_first']:
display_name = '%s %s' % (config['singular'], feature_name)
else:
display_name = '%s %s' % (feature_name, config['singular'])
Boundary.objects.create(
set=set,
kind=config['singular'],
external_id=external_id,
name=feature_name,
display_name=display_name,
metadata=metadata,
shape=geometry.wkt,
simple_shape=simple_geometry.wkt,
centroid=geometry.geos.centroid)
log.info('Saved %i %s.' % (set.count, kind))
