예제 #1
0
    def handle(self, *args, **options):
        # Load configuration
        sys.path.append(options['data_dir'])
        from definitions import SHAPEFILES

        if options['only']:
            only = options['only'].upper().split(',')
            # TODO: stripping whitespace here because optparse doesn't handle
            # it correctly
            sources = [
                s for s in SHAPEFILES if s.replace(' ', '').upper() in only
            ]
        elif options['except']:
            exceptions = options['except'].upper().split(',')
            # See above
            sources = [
                s for s in SHAPEFILES
                if s.replace(' ', '').upper() not in exceptions
            ]
        else:
            sources = [s for s in SHAPEFILES]

        for kind, config in SHAPEFILES.items():
            if kind not in sources:
                log.info('Skipping %s.' % kind)
                continue

            log.info('Processing %s.' % kind)

            self.load_set(kind, config, options)
    def handle(self, *args, **options):
        # Load configuration
        sys.path.append(options['data_dir'])
        from definitions import SHAPEFILES

        if options['only']:
            only = options['only'].upper().split(',')
            # TODO: stripping whitespace here because optparse doesn't handle 
            # it correctly
            sources = [s for s in SHAPEFILES
                       if s.replace(' ', '').upper() in only]
        elif options['except']:
            exceptions = options['except'].upper().split(',')
            # See above
            sources = [s for s in SHAPEFILES
                       if s.replace(' ', '').upper() not in exceptions]
        else:
            sources = [s for s in SHAPEFILES]

        for kind, config in SHAPEFILES.items():
            if kind not in sources:
                log.info('Skipping %s.' % kind)
                continue

            log.info('Processing %s.' % kind)

            self.load_set(kind, config, options)
    def handle(self, *args, **options):
        sys.path.append(os.path.join(settings.SITE_DIR, SHAPEFILES_DIR))
        from definitions import SHAPEFILES

        for name, options in SHAPEFILES.items():
            self.stdout.write('Processing \'{}\' now...'.format(name))

            self.stdout.write('Attempting to create collection...')
            collection, _ = Collection.objects.get_or_create(
                name=name,
                authority=options['authority'],
                last_updated=options['last_updated'],
                count=0,
                slug=options['slug'],
                source_url=options['source_url']
            )

            if _ is True:
                self.stdout.write('\'{}\' collection created!'.format(name))
            else:
                self.stdout.write('\'{}\' collection already exists!'
                                  ' Not a problem.'.format(name))

            lm = LayerMapping(Shape,
                              os.path.join(SHAPEFILES_DIR, options['file']),
                              options['layer_mapping'],
                              encoding='latin-1')

            lm.save(verbose=True, strict=True)
예제 #4
0
    def handle(self, *args, **options):
        sys.path.append(os.path.join(settings.SITE_DIR, SHAPEFILES_DIR))
        from definitions import SHAPEFILES

        for name, options in SHAPEFILES.items():
            self.stdout.write('Processing \'{}\' now...'.format(name))

            self.stdout.write('Attempting to create collection...')
            collection, _ = Collection.objects.get_or_create(
                name=name,
                authority=options['authority'],
                last_updated=options['last_updated'],
                count=0,
                slug=options['slug'],
                source_url=options['source_url'])

            if _ is True:
                self.stdout.write('\'{}\' collection created!'.format(name))
            else:
                self.stdout.write('\'{}\' collection already exists!'
                                  ' Not a problem.'.format(name))

            lm = LayerMapping(Shape,
                              os.path.join(SHAPEFILES_DIR, options['file']),
                              options['layer_mapping'],
                              encoding='latin-1')

            lm.save(verbose=True, strict=True)
    def handle(self, *args, **options):
        # Load configuration
        sys.path.append(options['data_dir'])
        from definitions import SHAPEFILES

        if options['only']:
            only = options['only'].split(',')
            # TODO: stripping whitespace here because optparse doesn't handle it correctly
            sources = [s for s in SHAPEFILES if s.replace(' ', '') in only]
        elif options['except']:
            exceptions = options['except'].upper().split(',')
            # See above
            sources = [s for s in SHAPEFILES if s.replace(' ', '') not in exceptions]
        else:
            sources = [s for s in SHAPEFILES]
        
        for kind, config in SHAPEFILES.items():
            if kind not in sources:
                log.info('Skipping %s.' % kind)
                continue

            log.info('Processing %s.' % kind)

            if options['clear']:
                set = None

                try:
                    set = BoundarySet.objects.get(name=kind)
                    if set:
                        log.info('Clearing old %s.' % kind)
                        set.boundaries.all().delete()
                        set.delete()
                        log.info('Loading new %s.' % kind)
                except BoundarySet.DoesNotExist:
                    log.info("No existing boundary set of kind [%s] so nothing to delete" % kind)

            path = os.path.join(options['data_dir'], config['file'])
            datasources = create_datasources(path)

            layer = datasources[0][0]

            # Create BoundarySet
            try:
                set = BoundarySet.objects.get(name=kind)
                log.info("Using existing BoundarySet [%s]" % set.slug)
            except:
                set = BoundarySet.objects.create(
                    name=kind,
                    singular=config['singular'],
                    kind_first=config['kind_first'],
                    authority=config['authority'],
                    domain=config['domain'],
                    last_updated=config['last_updated'],
                    href=config['href'],
                    notes=config['notes'],
                    count=len(layer),
                    metadata_fields=layer.fields)
                log.info("Created new BoundarySet [%s]" % set.slug)

            for datasource in datasources:
                log.info("Loading %s from %s" % (kind, datasource.name))
                # Assume only a single-layer in shapefile
                if datasource.layer_count > 1:
                    log.warn('%s shapefile [%s] has multiple layers, using first.' % (datasource.name, kind))
                layer = datasource[0]    
                self.add_boundaries_for_layer(config,layer,set)
            
            set.count = Boundary.objects.filter(set=set).count() # sync this with reality
            set.save()
            # TODO: work through additional shapefiles, increment set.count
            log.info('%s count: %i' % (kind, set.count))
예제 #6
0
#!/usr/bin/env python

import sys
from os.path import dirname, join, splitext
from subprocess import call

# add the shapefiles dir to the path
shapefiles_path = join(dirname(__file__), 'shapefiles')
sys.path.append(shapefiles_path)

# load our shapefile definitions
from definitions import SHAPEFILES

if 'geojson' in sys.argv:
    for title, data in SHAPEFILES.items():
        shapefile = join('shapefiles', data['file'])
        geojsonfile = splitext(shapefile)[0] + '.geojson'
        proc = call(['ogr2ogr', '-f', 'geoJSON', geojsonfile, shapefile])
        print "* %s [source](%s) [map](%s)" % (
            title, data["href"], geojsonfile)
예제 #7
0
    def handle(self, *args, **options):
        # Load configuration
        sys.path.append(os.path.abspath(SHAPEFILES_DIR))
        from definitions import SHAPEFILES

        if options['only']:
            only = options['only'].split(',')
            # TODO: stripping whitespace here because optparse doesn't handle it correclty
            sources = [s for s in SHAPEFILES if s.replace(' ', '') in only]
        elif options['except']:
            exceptions = options['except'].upper().split(',')
            # See above
            sources = [s for s in SHAPEFILES if s.replace(' ', '') not in exceptions]
        else:
            sources = [s for s in SHAPEFILES]
        
        # Get spatial reference system for the postgis geometry field
        geometry_field = Boundary._meta.get_field_by_name(GEOMETRY_COLUMN)[0]
        SpatialRefSys = connections[DEFAULT_DB_ALIAS].ops.spatial_ref_sys()
        db_srs = SpatialRefSys.objects.get(srid=geometry_field.srid).srs

        for kind, config in SHAPEFILES.items():
            if kind not in sources:
                log.info('Skipping %s.' % kind)
                continue

            log.info('Processing %s.' % kind)

            if options['clear']:
                set = None

                try:
                    set = BoundarySet.objects.get(name=kind)
                except BoundarySet.DoesNotExist:
                    pass

                if set:
                    log.info('Clearing old %s.' % kind)
                    set.boundaries.all().delete()
                    set.delete()
                    log.info('Loading new %s.' % kind)

            path = os.path.join(SHAPEFILES_DIR, config['file'])
            datasource = DataSource(path)

            # Assume only a single-layer in shapefile
            if datasource.layer_count > 1:
                log.warn('%s shapefile has multiple layers, using first.' % kind)

            layer = datasource[0]
            if 'srid' in config:
                layer_srs = SpatialRefSys.objects.get(srid=config['srid']).srs
            else:
                layer_srs = layer.srs

            # Create a convertor to turn the source data into
            transformer = CoordTransform(layer_srs, db_srs)

            # Create BoundarySet
            set = BoundarySet.objects.create(
                name=kind,
                singular=config['singular'],
                kind_first=config['kind_first'],
                authority=config['authority'],
                domain=config['domain'],
                last_updated=config['last_updated'],
                href=config['href'],
                notes=config['notes'],
                count=len(layer),
                metadata_fields=layer.fields)

            for feature in layer:
                # Transform the geometry to the correct SRS
                geometry = self.polygon_to_multipolygon(feature.geom)
                geometry.transform(transformer)
    
                # Create simplified geometry field by collapsing points within 1/1000th of a degree.
                # Since Chicago is at approx. 42 degrees latitude this works out to an margin of 
                # roughly 80 meters latitude and 112 meters longitude.
                # Preserve topology prevents a shape from ever crossing over itself.
                simple_geometry = geometry.geos.simplify(0.0001, preserve_topology=True)
                
                # Conversion may force multipolygons back to being polygons
                simple_geometry = self.polygon_to_multipolygon(simple_geometry.ogr)

                # Extract metadata into a dictionary
                metadata = {}

                for field in layer.fields:
                    
                    # Decode string fields using encoding specified in definitions config
                    if config['encoding'] != '':
                        try:
                            metadata[field] = feature.get(field).decode(config['encoding'])
                        # Only strings will be decoded, get value in normal way if int etc.
                        except AttributeError:
                            metadata[field] = feature.get(field)
                    else:
                        metadata[field] = feature.get(field)

                external_id = config['ider'](feature)
                feature_name = config['namer'](feature)
                
                # If encoding is specified, decode id and feature name
                if config['encoding'] != '':
                    external_id = external_id.decode(config['encoding'])
                    feature_name = feature_name.decode(config['encoding'])

                if config['kind_first']:
                    display_name = '%s %s' % (config['singular'], feature_name)
                else:
                    display_name = '%s %s' % (feature_name, config['singular'])

                Boundary.objects.create(
                    set=set,
                    kind=config['singular'],
                    external_id=external_id,
                    name=feature_name,
                    display_name=display_name,
                    metadata=metadata,
                    shape=geometry.wkt,
                    simple_shape=simple_geometry.wkt,
                    centroid=geometry.geos.centroid)

            log.info('Saved %i %s.' % (set.count, kind))
예제 #8
0
    def handle(self, *args, **options):
        # Load configuration
        sys.path.append(os.path.abspath(SHAPEFILES_DIR))
        from definitions import SHAPEFILES
        
        if options['only']:
            only = options['only'].split(',')
            # TODO: stripping whitespace here because optparse doesn't handle 
            # it correclty
            sources = [s for s in SHAPEFILES if s.replace(' ', '') in only]
        elif options['except']:
            exceptions = options['except'].upper().split(',')
            # See above
            sources = [s for s in SHAPEFILES if s.replace(' ', '') not in exceptions]
        else:
            sources = [s for s in SHAPEFILES]
        
        # Get spatial reference system for the postgis geometry field
        geometry_field = Boundary._meta.get_field_by_name(GEOMETRY_COLUMN)[0]
        SpatialRefSys = connections[DEFAULT_DB_ALIAS].ops.spatial_ref_sys()
        db_srs = SpatialRefSys.objects.get(srid=geometry_field.srid).srs

        for kind, config in SHAPEFILES.items():
            if kind not in sources:
                log.info('Skipping %s.' % kind)
                continue

            log.info('Processing %s.' % kind)

            if options['clear']:
                set = None

                try:
                    set = BoundarySet.objects.get(name=kind)
                except BoundarySet.DoesNotExist:
                    pass

                if set:
                    log.info('Clearing old %s.' % kind)
                    set.boundaries.all().delete()
                    set.delete()
                    log.info('Loading new %s.' % kind)

            path = os.path.join(SHAPEFILES_DIR, config['file'])
            datasource = DataSource(path)

            # Assume only a single-layer in shapefile
            if datasource.layer_count > 1:
                log.warn('%s shapefile has multiple layers, using first.' % kind)

            layer = datasource[0]
            if 'srid' in config and config['srid']:
                layer_srs = SpatialRefSys.objects.get(srid=config['srid']).srs
            else:
                layer_srs = layer.srs

            # Create a convertor to turn the source data into
            transformer = CoordTransform(layer_srs, db_srs)

            # Create BoundarySet
            set = BoundarySet.objects.create(
                name=kind,
                singular=config['singular'],
                kind_first=config['kind_first'],
                authority=config['authority'],
                domain=config['domain'],
                last_updated=config['last_updated'],
                href=config['href'],
                notes=config['notes'],
                count=len(layer),
                metadata_fields=layer.fields)

            for feature in layer:
                # Transform the geometry to the correct SRS
                geometry = self.polygon_to_multipolygon(feature.geom)
                geometry.transform(transformer)
    
                # Create simplified geometry field by collapsing points 
                # within 1/1000th of a degree. Since Chicago is at approx. 
                # 42 degrees latitude this works out to an margin of roughly 
                # 80 meters latitude and 112 meters longitude. Preserve 
                # topology prevents a shape from ever crossing over itself.
                simple_geometry = geometry.geos.simplify(0.0001, preserve_topology=True)
                
                # Conversion may force multipolygons back to being polygons
                simple_geometry = self.polygon_to_multipolygon(simple_geometry.ogr)

                # Extract metadata into a dictionary
                metadata = {}

                for field in layer.fields:
                    
                    # Decode string fields using encoding specified in 
                    # definitions config
                    if config['encoding'] != '':
                        try:
                            metadata[field] = feature.get(field).decode(config['encoding'])
                        # Only strings will be decoded, get value in normal way if int etc.
                        except AttributeError:
                            metadata[field] = feature.get(field)
                    else:
                        metadata[field] = feature.get(field)

                external_id = config['ider'](feature)
                feature_name = config['namer'](feature)
                
                # If encoding is specified, decode id and feature name
                if config['encoding'] != '':
                    external_id = external_id.decode(config['encoding'])
                    feature_name = feature_name.decode(config['encoding'])

                if config['kind_first']:
                    display_name = '%s %s' % (config['singular'], feature_name)
                else:
                    display_name = '%s %s' % (feature_name, config['singular'])

                Boundary.objects.create(
                    set=set,
                    kind=config['singular'],
                    external_id=external_id,
                    name=feature_name,
                    display_name=display_name,
                    metadata=metadata,
                    shape=geometry.wkt,
                    simple_shape=simple_geometry.wkt,
                    centroid=geometry.geos.centroid)

            log.info('Saved %i %s.' % (set.count, kind))