def load_zip(self, filename): path = os.path.abspath(filename) csv_path = os.path.join(os.path.dirname(path), 'plz_einwohner.csv') with open(csv_path) as f: reader = csv.DictReader(f) population = {x['plz']: int(x['einwohner']) for x in reader} ds = DataSource(path) mapping = LayerMapping(GeoRegion, ds, {'geom': 'geometry'}) layer = ds[0] count = float(len(layer)) for i, feature in enumerate(layer): self.stdout.write('%.2f%%\r' % (i / count * 100), ending='') name = feature['plz'].as_string() slug = name geom = mapping.feature_kwargs(feature)['geom'] GeoRegion.objects.update_or_create( slug=slug, kind='zipcode', defaults={ 'name': name, 'geom': geom, 'description': feature['note'].as_string(), 'region_identifier': name, 'global_identifier': 'DE-%s' % name, 'area': feature.geom.area, 'population': population.get(name, None), 'level': 3, 'valid_on': timezone.now() } )
def load_boroughs(self, path, parent=None, name_col=None, ident_col=None): ds = DataSource(path) mapping = LayerMapping(GeoRegion, ds, {'geom': 'POLYGON'}) layer = ds[0] parent = GeoRegion.objects.get(kind='municipality', name=parent) for i, feature in enumerate(layer): name = feature[name_col].as_string() identifier = feature[ident_col].as_string()[:2] kind = 'borough' kind_detail = 'Bezirk' slug = slugify(name) geom = mapping.feature_kwargs(feature)['geom'] region_identifier = parent.region_identifier + identifier GeoRegion.objects.update_or_create( slug=slug, kind=kind, defaults={ 'name': name, 'kind': kind, 'kind_detail': kind_detail, 'level': 6, 'region_identifier': region_identifier, 'global_identifier': '', 'population': None, 'geom': geom, 'area': feature.geom.area, 'valid_on': None, 'part_of': parent } )
def handle(self, **options): shapefile = os.path.join(settings.HERE, os.pardir, 'data', 'TM_WORLD_BORDERS-0.3.shp') ds = DataSource(shapefile) print len(ds), "layer" layer = ds[0] print '%s contains %s geometries (type: %s)' % (layer, len(layer), layer.geom_type) print layer.srs #for feature in layer: # print feature.get('NAME'), feature.geom.num_points mapping = { 'fips': 'FIPS', 'iso2': 'ISO2', 'iso3': 'ISO3', 'un': 'UN', 'name': 'NAME', 'area': 'AREA', 'pop2005': 'POP2005', 'region': 'REGION', 'subregion': 'SUBREGION', 'lon': 'LON', 'lat': 'LAT', 'mpoly': 'MULTIPOLYGON', } lm = LayerMapping(Country, shapefile, mapping, transform=True, encoding='iso-8859-1') # Antartica doesn't convert to Spherical Mercator lm.save(fid_range=(1, 144), verbose=True) lm.save(fid_range=(146, 246), verbose=True)
def handle(self, *args, **options): datadir = args[0] # Need to do this 50 times, onece for the shapefile of each state for i in range(1,57): # files are named with zero padded FIPS codes if i < 10: padded_i = "0"+str(i) else: padded_i = str(i) shpfile = os.path.abspath(os.path.join(os.path.dirname(__file__), datadir+'/state_tract_shapefiles/tl_2010_'+padded_i+'_tract10/tl_2010_'+padded_i+'_tract10.shp')) print "Attempting import of shapefile "+shpfile try: lm = LayerMapping(CensusTract, shpfile, censustracts_mapping) except OGRException: print "Could not open datasource ", print shpfile else: try: lm.save(strict=True, verbose=False, progress=True) except IntegrityError: print "Already imported ", print shpfile from django.db import transaction transaction.rollback() else: print "Imported shapefile "+shpfile print
def handle(self, *args, **options): self.stdout.write('Import USA State') state_shp = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../shpData/southStates.shp')) state_mapping = {'statefp' : 'STATEFP','name' : 'NAME','geom' : 'POLYGON25D', } lm = LayerMapping(State, state_shp, state_mapping,transform=False, encoding='iso-8859-1') lm.save(strict=True, verbose=True) self.stdout.write('Import OK')
def populate_GADM( model, shp_file, mapping_dict, verbose=False ): """ Populates a model of GADM family Parameters ---------- model : <model> shp_file : str mapping_dict : dict verbose : bool """ try: lm = LayerMapping( model, shp_file, mapping_dict, transform=False, ) with transaction.atomic(): lm.save(verbose=verbose) except IntegrityError as ie: print "Integrity error:", ie.message except Exception as e: print 'Error loading GADM data!' print e.message
def load_parcels(shapefile_path, verbose=False, progress=True, silent=False, stream=None): """ Load parcels from shapefile at given path. Raises IntegrityError on a duplicate PL, and rolls back the load. Returns number of parcels loaded. """ # monkeypatch no-op transaction handling into LayerMapping, as we # wrap it in a transaction including more operations. LayerMapping.TRANSACTION_MODES['none'] = lambda func: func # first delete existing parcels Parcel.objects.all().delete() lm = LayerMapping( get_parcel_proxy(datetime.datetime.now()), shapefile_path, parcel_mapping, transform=True, transaction_mode='none') lm.save( strict=True, verbose=verbose, progress=progress, silent=silent, stream=stream or sys.stdout) return Parcel.objects.count()
def handle(self, *args, **options): sys.path.append(os.path.join(settings.SITE_DIR, SHAPEFILES_DIR)) from definitions import SHAPEFILES for name, options in SHAPEFILES.items(): self.stdout.write('Processing \'{}\' now...'.format(name)) self.stdout.write('Attempting to create collection...') collection, _ = Collection.objects.get_or_create( name=name, authority=options['authority'], last_updated=options['last_updated'], count=0, slug=options['slug'], source_url=options['source_url'] ) if _ is True: self.stdout.write('\'{}\' collection created!'.format(name)) else: self.stdout.write('\'{}\' collection already exists!' ' Not a problem.'.format(name)) lm = LayerMapping(Shape, os.path.join(SHAPEFILES_DIR, options['file']), options['layer_mapping'], encoding='latin-1') lm.save(verbose=True, strict=True)
def handle(self, *args, **options): """ Load community shapefile using LayerMapping; automatically checks projection, if necessary transforms to WSG 1984 """ neighborhood_mapping = { 'boundary': 'MULTIPOLYGON', 'primary_name': 'COMMUNITY', 'secondary_name': 'AREA_NUM_1', } path_to_shp = 'data/chicago_communities/CommAreas.shp' lm = LayerMapping(Neighborhood, path_to_shp, neighborhood_mapping) self.check_neighborhood_table() lm.save(strict=True) self.stdout.write('Successfully loaded %s communities from %s layer(s) into database\n' % (len(lm.ds[0]), lm.ds.layer_count)) # Change case of imported name strings from UPPER to Caps Case communities = Neighborhood.objects.all() for community in communities: names = [name.capitalize() for name in community.primary_name.split()] primary_name_caps = " ".join(names) self.stdout.write('Changing name %s ==> %s\n' % (community.primary_name, primary_name_caps)) community.primary_name = primary_name_caps community.save() # Export topojson export_topojson.Command().handle()
def from_shapefile(strict=False, progress=True, verbose=False, **kwargs): """ Load parcel data into the database from the processed shapefile. """ parcel_shp = get_processed_data_file(os.path.join('parcels', 'parcels.shp')) mapping = LayerMapping(Parcel, parcel_shp, parcel_mapping, transform=False) mapping.save(strict=strict, progress=progress, verbose=verbose, **kwargs)
def shp(): """ Load the ESRI shapefile from the Census in the County model. Example usage: >> from us_counties import load; load.shp(); """ # Import the database model where we want to store the data from models import County # A crosswalk between the fields in our database and the fields in our # source shapefile shp2db = { 'polygon_4269': 'Polygon', 'state_fips_code': 'STATEFP', 'county_fips_code': 'COUNTYFP', 'fips_code': 'CNTYIDFP', 'county_ansi_code': 'COUNTYNS', 'short_name': 'NAME', 'full_name': 'NAMELSAD', 'csa_code': 'CSAFP', 'msa_code': 'CBSAFP', 'mda_code': 'METDIVFP', 'functional_status': 'FUNCSTAT', } # Load our model, shape, and the map between them into GeoDjango's magic # shape loading function (I also slipped the source coordinate system in # there. The Census says they put everything in NAD 83, which translates # to 4269 in the SRID id system.) lm = LayerMapping(County, shp_file, shp2db, source_srs=4269, encoding='latin-1') # Fire away! lm.save(verbose=False)
def run(verbose=True): lm = LayerMapping(Voie, tunisia_voie, voie_mapping, transform=True, encoding='utf8') lm.save(strict=True, verbose=verbose) for i in Voie.objects.all(): i.label = slugify(i.label ) i.nom_usage = slugify(i.nom_usage) i.save()
def handle(self, shapefile, manipulator, *args, **options): try: manip_model = get_class(manipulator) except: raise Exception("The %s model could not be found. \nBe sure and provide the complete description: <module name>.models.<manipulator model name>" %manipulator) ds = DataSource(shapefile) if len(ds) != 1: raise Exception("Data source should only contain a single layer. Aborting.") layer = ds[0] if len(layer) != 1: raise Exception("Layer should containing ONLY a single feature") if not 'polygon' in layer.geom_type.name.lower(): print layer.geom_type.name raise Exception("This geometry must be a polygon") mapping = {'geometry': 'MULTIPOLYGON'} lm = LayerMapping(manip_model, shapefile, mapping) lm.save() manip_geom = manip_model.objects.order_by('-creation_date')[0] if options.get('region_name'): manip_geom.name = options.get('region_name') manip_geom.save() else: manip_geom.name = layer.name manip_geom.save() print "" print "The manipulaotr geometry, %s, has been added to the %s model with primary key = %s" % (manip_geom.name, manipulator, manip_geom.pk) print "To switch to this geometry, you will need to run 'manage.py change_manipulator_geom %s %s'" % (manip_geom.pk, manipulator) print ""
def county_import(county_shp, year): if year == "2010": county_mapping = { 'state_fips_code': 'STATEFP10', 'fips_code': 'COUNTYFP10', 'county_identifier': 'GEOID10', 'name': 'NAME10', 'name_and_description': 'NAMELSAD10', 'legal_statistical_description': 'LSAD10', 'fips_55_class_code': 'CLASSFP10', 'feature_class_code': 'MTFCC10', 'functional_status': 'FUNCSTAT10', 'mpoly': 'POLYGON', } else: county_mapping = { 'state_fips_code': 'STATEFP', 'fips_code': 'COUNTYFP', 'county_identifier': 'GEOID', 'name': 'NAME', 'name_and_description': 'NAMELSAD', 'legal_statistical_description': 'LSAD', 'fips_55_class_code': 'CLASSFP', 'feature_class_code': 'MTFCC', 'functional_status': 'FUNCSTAT', 'mpoly': 'POLYGON', } lm = LayerMapping(County, county_shp, county_mapping, encoding='LATIN1') lm.save(verbose=True)
def handle(self, *args, **options): mapping = { 'perimeter': 'PERIMETER', 'ward': 'WARD', 'alderman': 'ALDERMAN', 'ward_phone': 'WARD_PHONE', 'hall_phone': 'HALL_PHONE', 'hall_office': 'HALL_OFFIC', 'address': 'ADDRESS', 'edit_date1': 'EDIT_DATE1', 'shape_area': 'SHAPE_AREA', 'shape_len': 'SHAPE_LEN', 'geom': 'MULTIPOLYGON' } shp_data = requests.get(SHP) if shp_data.status_code != 200: raise CommandError('City data portal returned a %s status when downloading Wards shapefile: %s' % (shp_data.status_code, shp_data.content)) else: s = StringIO(shp_data.content) z = zipfile.ZipFile(s) data_path = os.path.join(os.path.curdir,'data/shp/wards') fname = [f for f in z.namelist() if f.endswith('shp')][0] z.extractall(data_path) datafile = os.path.join(data_path, fname) lm = LayerMapping(Ward, datafile, mapping) lm.save(strict=True,progress=True)
def test05_geography_layermapping(self): "Testing LayerMapping support on models with geography fields." # There is a similar test in `layermap` that uses the same data set, # but the County model here is a bit different. if not gdal.HAS_GDAL: return from django.contrib.gis.utils import LayerMapping # Getting the shapefile and mapping dictionary. shp_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "../", "data")) co_shp = os.path.join(shp_path, "counties", "counties.shp") co_mapping = {"name": "Name", "state": "State", "mpoly": "MULTIPOLYGON"} # Reference county names, number of polygons, and state names. names = ["Bexar", "Galveston", "Harris", "Honolulu", "Pueblo"] num_polys = [1, 2, 1, 19, 1] # Number of polygons for each. st_names = ["Texas", "Texas", "Texas", "Hawaii", "Colorado"] lm = LayerMapping(County, co_shp, co_mapping, source_srs=4269, unique="name") lm.save(silent=True, strict=True) for c, name, num_poly, state in zip(County.objects.order_by("name"), names, num_polys, st_names): self.assertEqual(4326, c.mpoly.srid) self.assertEqual(num_poly, len(c.mpoly)) self.assertEqual(name, c.name) self.assertEqual(state, c.state)
def test05_geography_layermapping(self): "Testing LayerMapping support on models with geography fields." # There is a similar test in `layermap` that uses the same data set, # but the County model here is a bit different. if not gdal.HAS_GDAL: return from django.contrib.gis.utils import LayerMapping # Getting the shapefile and mapping dictionary. shp_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', 'data')) co_shp = os.path.join(shp_path, 'counties', 'counties.shp') co_mapping = {'name' : 'Name', 'state' : 'State', 'mpoly' : 'MULTIPOLYGON', } # Reference county names, number of polygons, and state names. names = ['Bexar', 'Galveston', 'Harris', 'Honolulu', 'Pueblo'] num_polys = [1, 2, 1, 19, 1] # Number of polygons for each. st_names = ['Texas', 'Texas', 'Texas', 'Hawaii', 'Colorado'] lm = LayerMapping(County, co_shp, co_mapping, source_srs=4269, unique='name') lm.save(silent=True, strict=True) for c, name, num_poly, state in zip(County.objects.order_by('name'), names, num_polys, st_names): self.assertEqual(4326, c.mpoly.srid) self.assertEqual(num_poly, len(c.mpoly)) self.assertEqual(name, c.name) self.assertEqual(state, c.state)
def handle(self, shapefile, *args, **options): ds = DataSource(shapefile) if len(ds) != 1: raise Exception("Data source should only contain a single layer. Aborting.") layer = ds[0] if len(layer) != 1: raise Exception("Layer should containing ONLY a single feature") if not 'polygon' in layer.geom_type.name.lower(): print layer.geom_type.name raise Exception("Study region must be a multigeometry") if options.get('region_name'): mapping = { 'geometry': 'MULTIPOLYGON', } else: mapping = { 'geometry': 'MULTIPOLYGON', 'name': 'name', } lm = LayerMapping(StudyRegion, shapefile, mapping, transform=False) lm.save() study_region = StudyRegion.objects.order_by('-creation_date')[0] if options.get('region_name'): study_region.name = options.get('region_name') study_region.save() print "" print "Study region created: %s, primary key = %s" % (study_region.name, study_region.pk) print "To switch to this study region, you will need to run 'python manage.py change_study_region %s'" % (study_region.pk, ) print ""
def run(self): if self.overwrite: self.print('Removing existing neighborhoods...', end='') if self.real_run: Neighborhood.objects.all().delete() self.print('Done') # Disconnect this temporarily so that the ``unique='name'`` option # passed to LayerMapping will work correctly (it collects all # records from the RLIS neighborhoods shapefile with the same name # into a single database record; if we normalize the names on save, # this feature won't work). pre_save.disconnect(normalize_name, sender=Neighborhood) self.print('Adding neighborhoods...', end='') if self.real_run: mapping = LayerMapping( Neighborhood, self.path, Neighborhood.layer_mapping, source_srs=self.from_srid, unique='name') mapping.save(strict=True) self.print('Done') self.print('Normalizing neighborhood names...', end='') neighborhoods = Neighborhood.objects.all() if self.real_run: with transaction.atomic(): for neighborhood in neighborhoods: neighborhood.name = Neighborhood.normalize_name(neighborhood.name) neighborhood.save() self.print('Done') pre_save.connect(normalize_name, sender=Neighborhood)
def handle(self, *args, **options): world_mapping = { 'fips' : 'FIPS', 'iso2' : 'ISO2', 'iso3' : 'ISO3', 'un' : 'UN', 'name' : 'NAME', 'area' : 'AREA', 'pop2005' : 'POP2005', 'region' : 'REGION', 'subregion' : 'SUBREGION', 'lon' : 'LON', 'lat' : 'LAT', 'mpoly' : 'MULTIPOLYGON', } user = User.objects.get(id=1) world_shp = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data', 'TM_WORLD_BORDERS-0.3.shp')) with transaction.commit_on_success(): lm = LayerMapping(WorldBorder, world_shp, world_mapping, transform=False, encoding='iso-8859-1') lm.save(strict=True, verbose=True) for country in WorldBorder.objects.all(): gr = GeospatialReference( title=country.name, address=country.name, geometry=country.mpoly, point=Point(country.lon, country.lat), description='http://thematicmapping.org/', user=user, ) gr.save()
def run(verbose=True): ds = Dataset(name = '2010 Census Tracts', cached = datetime.utcnow().replace(tzinfo=utc), cache_max_age = 1000, remote_id_field = 'GEOID10', name_field = 'NAMELSAD10', lat_field = 'INTPTLAT10', lon_field = 'INTPTLON10', field1_en = 'Land Area', field1_name = 'ALAND10', field2_en = 'Water Area', field2_name = 'AWATER10') tract_mapping = { 'remote_id' : ds.remote_id_field, 'name' : ds.name_field, 'lat' : ds.lat_field, 'lon' : ds.lon_field, 'field1' : ds.field1_name, 'field2' : ds.field2_name, 'mpoly' : 'MULTIPOLYGON', } tract_shp = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data/tl_2010_36_tract10.shp')) lm = LayerMapping(MapPolygon, tract_shp, tract_mapping, transform=False, encoding='iso-8859-1') lm.save(strict=True, verbose=verbose) ds.save() MapPolygon.objects.filter(dataset = None).update(dataset = ds)
def handle(self, *args, **options): verbose = options.get('verbose') source = options.get('source') if verbose: console = self.stdout else: console = None if not source: print "Source must be specified" return 0 from django.contrib.gis.gdal import DataSource ds = DataSource("PG:" "host=localhost port=5432 dbname=ex1 user=ex1 password=ex1") lyr_ind = -1 for i in range(len(ds)): if str(ds[i]) == source: lyr_ind = i break if lyr_ind == -1: print "Could not find table in data source" return 0 lm = LayerMapping( Country, ds, COUNTRY_MAPPING, layer=lyr_ind, encoding='iso-8859-1') lm.save(strict=True, verbose=verbose)
def run(verbose=True): # args are model name, source file, and mapping dict # Transform is false because the database format is SRID 4326, which is WGS84, # and the shape file is already in this format lm = LayerMapping(WorldBorder, world_shp, world_mapping, transform=False, encoding='iso-8859-1') lm.save(strict=True, verbose=verbose)
def handle(self, *args, **options): for shp in args: lm = LayerMapping(self.config[shp]['model'], self.config[shp]['file'], self.config[shp]['mapping'], encoding='iso-8859-1') lm.save(strict=True, verbose=True) self.stdout.write('Successfully imported "%s"\n' % shp)
def fill_up_db(shapefile, verbose=False): from django.contrib.gis.utils import LayerMapping mapping = { 'name': 'NAME', 'area_code': 'AREA_CODE', 'descriptio': 'DESCRIPTIO', 'file_name': 'FILE_NAME', 'number': 'NUMBER', 'number0': 'NUMBER0', 'polygon_id': 'POLYGON_ID', 'unit_id': 'UNIT_ID', 'code': 'CODE', 'hectares': 'HECTARES', 'area': 'AREA', 'type_code': 'TYPE_CODE', 'descript0': 'DESCRIPT0', 'type_cod0': 'TYPE_COD0', 'descript1': 'DESCRIPT1', 'geom': 'POLYGON', } lm = LayerMapping(Region, shapefile, mapping, transform=True, encoding='iso-8859-1') lm.save(strict=True, verbose=verbose) Region.objects.filter(descriptio__icontains='Welsh Assembly').delete() print "Regions imported"
def shp(): """ Load the ESRI shapefile from the Census in the District model. Example usage: >> from congressional_districts import load; load.shp(); """ # Import the database model where we want to store the data from models import District # A crosswalk between the fields in our database and the fields in our # source shapefile shp2db = { 'state_fips_code' : 'STATE', 'district_number' : 'CD', 'lsad' : 'LSAD', 'name' : 'NAME', 'lsad_trans' : 'LSAD_TRANS', 'polygon_4269' : 'POLYGON', } # Load our model, shape, and the map between them into GeoDjango's magic # shape loading function (I also slipped the source coordinate system in # there. The Census says they put everything in NAD 83, which translates # to 4269 in the SRID id system.) lm = LayerMapping(District, shp_file, shp2db, source_srs=4269, encoding='latin-1') # Fire away! lm.save(verbose=False)
def run(verbose=True): lm = LayerMapping(Noeud, tunisia_nodes, node_mapping, transform=True, encoding='utf8') lm.save(strict=True, verbose=verbose) for i in Noeud.objects.all(): i.type = slugify(i.type ) i.label = slugify(i.label) i.save()
def county_import(county_shp): if ( "2015" in county_shp or "2014" in county_shp or "2013" in county_shp or "2012" in county_shp or "2011" in county_shp ): county_mapping = { "state_fips_code": "STATEFP", "fips_code": "COUNTYFP", "county_identifier": "GEOID", "name": "NAME", "name_and_description": "NAMELSAD", "legal_statistical_description": "LSAD", "fips_55_class_code": "CLASSFP", "feature_class_code": "MTFCC", "functional_status": "FUNCSTAT", "mpoly": "POLYGON", } else: county_mapping = { "state_fips_code": "STATEFP10", "fips_code": "COUNTYFP10", "county_identifier": "GEOID10", "name": "NAME10", "name_and_description": "NAMELSAD10", "legal_statistical_description": "LSAD10", "fips_55_class_code": "CLASSFP10", "feature_class_code": "MTFCC10", "functional_status": "FUNCSTAT10", "mpoly": "POLYGON", } lm = LayerMapping(County, county_shp, county_mapping, encoding="LATIN1") lm.save(verbose=True)
def pre_save_layer(instance, sender, **kwargs): """Save to postgis if there is a datastore. """ # Abort if a postgis DATABASE is not configured. if DYNAMIC_DATASTORE not in settings.DATABASES: return # Do not process if there is no table. base_file = instance.get_base_file()[0] if base_file is None or base_file.name != 'shp': return filename = base_file.file.path # Load the table in postgis and get a mapping from fields in the database # and fields in the Shapefile. mapping = file2pgtable(filename, instance.name) # Get a dynamic model with the same name as the layer. model_description, __ = ModelDescription.objects.get_or_create( name=instance.name) # Set up the fields with the postgis table generate_model(model_description, mapping, db_key=DYNAMIC_DATASTORE) # Get the new actual Django model. TheModel = model_description.get_django_model() # Use layermapping to load the layer with geodjango lm = LayerMapping(TheModel, filename, mapping, encoding=instance.charset, using=DYNAMIC_DATASTORE, transform=None ) lm.save()
def run(verbose=True): if len(Neighborhood.objects.all()) == 0: lm = LayerMapping(Neighborhood, neighborhood_shape, mapping) lm.save(strict=True, verbose=verbose) Neighborhood.objects.exclude(city='Seattle').delete() print "There are %d neighborhoods in Seattle" % len(Neighborhood.objects.all()) else: print "Neighborhood shapefile already loaded (at least one neighborhood exists). Delete existing records via admin tool"
def handle(self, *args, **options): wb_count = WorldBorder.objects.count() if wb_count: self.stdout.write( '{} countries already exist'.format(wb_count)) return lm = LayerMapping( WorldBorder, WORLD_SHAPEFILE, WORLD_SHAPEFILE_MODEL_MAPPING, transform=False, encoding='iso-8859-1' ) lm.save(strict=True, verbose=False) self.stdout.write("Loaded world borders")
def handle(self, *args, **kwargs): wc_count = Constituency.objects.count() if wc_count: self.stdout.write( self.style.WARNING( '{} constituencies already exist'.format(wc_count))) return lm = LayerMapping(Constituency, constituency_shp, constituency_mapping, transform=False, encoding='iso-8859-1') lm.save(strict=True, verbose=False) self.stdout.write( self.style.SUCCESS("Loaded constituencies shapefile"))
def run(verbose=True): numfeatures_veredas=32305 #lm = LayerMapping(WorldBorder, ds, world_mapping, transform=False) #lm.save(strict=True, verbose=verbose) lm2 = LayerMapping(veredas, ds2, veredas_mapping, transaction_mode= 'autocommit',transform=False) lm2.save(strict=False, fid_range=(0,5000),verbose=False,progress=True,step=500) lm2.save(strict=False,fid_range=(5001,15000), verbose=False,progress=True,step=500) lm2.save(strict=False,fid_range=(15001,20000), verbose=False,progress=True,step=500)
def border_data(apps, schema_editor): # This border data sourced from iMapBC, specifically the "Province of BC - ABMS - Outlined" layer. # Shapefile inspected using: # ogrinfo -so gwells/migrations/ABMS_PROVINCE_SP/ABMS_PROV_polygon.shp ABMS_PROV_polygon # Model and mapping generated using: # python manage.py ogrinspect gwells/migrations/ABMS_PROVINCE_SP/ABMS_PROV_polygon.shp gwells.Border \ # --srid=4269 --mapping --multi Border = apps.get_model('gwells', 'Border') tmp_path = '/tmp/BCGW_ABMS_PROV-migrations/' zip_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'migrations/BCGW_ABMS_PROV.zip') with ZipFile(zip_path, 'r') as zipObj: zipObj.extractall(tmp_path) border_shp = os.path.join(tmp_path, 'ABMS_PROVINCE_SP/ABMS_PROV_polygon.shp') if not os.path.exists(border_shp): raise FileNotFoundError('file not found: {}'.format(border_shp)) border_mapping = { 'se_a_c_flg': 'SE_A_C_FLG', 'obejctid': 'OBEJCTID', 'shape': 'SHAPE', 'length_m': 'LENGTH_M', 'oic_number': 'OIC_NUMBER', 'area_sqm': 'AREA_SQM', 'upt_date': 'UPT_DATE', 'upt_type': 'UPT_TYPE', 'chng_org': 'CHNG_ORG', 'aa_parent': 'AA_PARENT', 'aa_type': 'AA_TYPE', 'aa_id': 'AA_ID', 'aa_name': 'AA_NAME', 'abrvn': 'ABRVN', 'bdy_type': 'BDY_TYPE', 'oic_year': 'OIC_YEAR', 'afctd_area': 'AFCTD_AREA', 'geom': 'MULTIPOLYGON25D', } lm = LayerMapping(Border, border_shp, border_mapping, transform=False) lm.save(strict=True, verbose=True) # Now clean up shutil.rmtree(tmp_path)
def handle(self, *args, **options): self.stdout.write('Import USA State') state_shp = os.path.abspath( os.path.join(os.path.dirname(__file__), '../../../shpData/southStates.shp')) state_mapping = { 'statefp': 'STATEFP', 'name': 'NAME', 'geom': 'POLYGON25D', } lm = LayerMapping(State, state_shp, state_mapping, transform=False, encoding='iso-8859-1') lm.save(strict=True, verbose=True) self.stdout.write('Import OK')
def run(verbose=True): """ load data from shp files to AlderDistrict table to reload data, rename shp files in format district.dbf district.prj district.shp district.shx """ lm = LayerMapping( AlderDistrict, DISTRICT_SHP, DISTRICT_MAPPING, transform=False, encoding='iso-8859-1', ) lm.save(strict=True, verbose=verbose)
def load_gpx_lm(verbose=True): """Recorre la carpeta datacentre/data/gpx e introduce en la BD los tracks de los GPX + dtours asociados Hace uso de LayerMapping --> Docs en https://docs.djangoproject.com/en/3.1/ref/contrib/gis/layermapping/ """ dir_gpx_data = Path( "/home/eguiwow/github/BIC/bic/datacentre/data/gpx" ) # Directorio donde están de momento guardados los GPXs for filepath in dir_gpx_data.glob('*.gpx'): print(filepath) lm = LayerMapping( Track, str(filepath), track_mapping, layer=2, transform=False) # Layer = 2 -> Layer = track de un GPX lm.save( strict=False, verbose=verbose ) # strict = True antes. Esto lo que hace es que si hay un fallo para la ejecución
def run(verbose=True): "Making stuff happen but this line is here because some/all stuff below here might get commented out." # from .models import TsunamiZone # lm2 = LayerMapping(TsunamiZone, tsunami_shp, tsunamizone_mapping, # transform=True, encoding='iso-8859-1', # unique=['typeid']) # lm2.save(strict=True, verbose=verbose) from .models import ImpactZoneData lm3 = LayerMapping(ImpactZoneData, impact_shp, impactzone_mapping, transform=True, encoding='iso-8859-1', unique=['zoneid']) lm3.save(strict=True, verbose=verbose)
def handle(self, *args, **options): if options['truncate']: self.w('Truncate zones table...', ending='') FirefightersZone.objects.all().delete() self.w(self.style.SUCCESS('Done.')) mapping = { 'name': 'BWZone', # The 'name' model field maps to the 'BWZone' layer field. 'mpolygon': 'geometry', } lm = LayerMapping(FirefightersZone, options['path_to_datasource'], mapping, transaction_mode='autocommit') lm.save(stream=self.stdout, verbose=True)
def run(verbose=True): for entry in data: file_name = os.path.abspath( os.path.join(os.path.dirname(__file__), 'data', f'{entry["data_file"]}.shp'), ) old_data = entry["data_class"].objects.all() for item in old_data: item.delete() print("-" * 80) print(f"{file_name}\n{entry['data_class']}\n{entry['mapping']}") lm = LayerMapping(entry["data_class"], file_name, entry["mapping"], transform=False) lm.save(strict=True, verbose=verbose)
def import_shapes(shapefile_path, logger): srs = SpatialReference(srs_wkt) lm = LayerMapping(Parcel, shapefile_path, { "shape_leng": "SHAPE_Leng", "shape_area": "SHAPE_Area", "map_par_id": "MAP_PAR_ID", "loc_id": "LOC_ID", "poly_type": "POLY_TYPE", "map_no": "MAP_NO", "source": "SOURCE", "plan_id": "PLAN_ID", "last_edit": "LAST_EDIT", "town_id": "TOWN_ID", "shape": "POLYGON" }, source_srs=srs) lm.save(strict=True)
def do_import(self): """ Imports objects into database """ self.stdout.write(self.style.MIGRATE_HEADING('\nBeginning import:')) region_mapping = {'name': 'NAME', 'polygon': 'POLYGON'} region_shp = '../data/kx-nz-regional-councils-2012-yearly-pattern-SHP/nz-regional-councils-2012-yearly-pattern.shp' lm = LayerMapping( Region, region_shp, region_mapping, transform=False, encoding='utf-8', ) lm.save(strict=True, verbose=False) self.stdout.write(self.style.SUCCESS('\nImport complete'))
def run(verbose=True): BUCKET_NAME = 'hacko-data-archive' KEY = '2018-housing-affordability/data/permits/' s3 = boto3.resource('s3') f = 'Residential_Building_Permits.geojson.json' file_path = '/data/permits/{}'.format(f) if not os.path.isfile(file_path): key = KEY + f s3.Bucket(BUCKET_NAME).download_file(key, file_path) PermitData.objects.all().delete() lm = LayerMapping(PermitData, file_path, mapping, transform=False, encoding='iso-8859-1') lm.save(strict=True, verbose=verbose)
def prep_layer_mapping(shpfile_name, model, mapping): shpfile = os.path.abspath( os.path.join(os.path.dirname(__file__), 'data', shpfile_name)) lm = LayerMapping(model, shpfile, mapping, transform=False, encoding='iso-8859-1') return lm
def load(): # # OB_districtshp = '/home/mohab/Main Folder/Airport/nash/Airport/Obied Airport/Tubra/Shapefiles/Ob_districts_fin.shp' # # lm1 = LayerMapping(Obeid_districts,OB_districtshp,OB_Districts_mapping,transform=True,encoding='utf-8') # # lm1.save() OB_streets = '/home/mohab/Main Folder/Airport/nash/Airport/Obied Airport/Tubra/Shapefiles/Ob_streets.shp' lm2 = LayerMapping(Obeid_streets, OB_streets, Obeid_streets_mapping, transform=True, encoding='utf-8') lm2.save() # OB_urban_area = '/home/mohab/Main Folder/Airport/nash/Airport/Obied Airport/Tubra/Shapefiles/Ob_urban_area.shp' # lm3 = LayerMapping(Ob_urban_area,OB_urban_area,ob_urban_area_mapping,transform=True,encoding='utf-8') # lm3.save() # import this file and run the load function in the shell
def handle(self, *args, **options): if options['geolevel']: # swap keys and values in TERRITORIO # use --geolevel=regione geolevels = dict( (value.lower(), key) for key, value in dict(Territorio.TERRITORIO).iteritems()) geolevel = geolevels[options['geolevel'].lower()] def _set_geolevel(sender, instance, **kwargs): """ Signal callback to set the geolevel for a place """ instance.territorio = geolevel pre_save.connect(_set_geolevel, sender=Territorio) # based, on geolevel, define source, mapping and reference system ID if geolevel == 'C': shapefile = self.comune_shp mapping = self.comune_mapping source_srs = 23032 elif geolevel == 'P': shapefile = self.provincia_shp mapping = self.provincia_mapping source_srs = 23032 elif geolevel == 'R': shapefile = self.regione_shp mapping = self.regione_mapping source_srs = 900913 else: raise CommandError( "supported geolevels: regione, provincia or comune") lm = LayerMapping(Territorio, shapefile, mapping, encoding=options['encoding'], source_srs=source_srs) lm.save(verbose=True) else: raise CommandError("geolevel must be specified")
def import_country(shp='evaluations/data/countries.shp'): """ Import the countries from a shapefile. """ # first delete all features (if any) Country.objects.all().delete() # mapping model-shp country_mapping = { 'name': 'NAME', 'geometry': 'MULTIPOLYGON', } # import features from shapefile to model countries = LayerMapping(Country, shp, country_mapping, transform=False, encoding='iso-8859-1') countries.save(verbose=True, progress=True) print 'Import completed.'
def handle(self, *args, **options): global znacka_id, status_id, author_id, import_fields, table_name znacka_id = options['znacka_id'] status_id = options['status_id'] author_id = options['author_id'] data_source = DataSource(args[0]) self.print_layer_info(data_source) fields = data_source[0].fields geom_type = data_source[0].geom_type table_name = data_source[0].name if geom_type == 'Point': mapping = { 'geom' : 'POINT', } if geom_type == "LineString": mapping = { 'geom' : 'LINESTRING', } if geom_type == "Polygon": mapping = { 'geom' : 'POLYGON', } for field in fields: if field == 'popis': mapping['desc'] = 'popis' continue if field == 'nazev': mapping['nazev'] = 'nazev' continue mapping['sit_' + field] = field import_fields = fields source_srs = "+proj=krovak +lat_0=49.5 +lon_0=24.83333333333333 +alpha=30.28813972222222 \ +k=0.9999 +x_0=0 +y_0=0 +ellps=bessel +pm=greenwich +units=m +no_defs \ +towgs84=570.8,85.7,462.8,4.998,1.587,5.261,3.56" encoding = "WINDOWS-1250" lm = LayerMapping(ImportPoi, args[0], mapping, source_srs = source_srs, encoding = encoding) lm.save(verbose=True)
def load_gpkg_lyr_to_model(model, model_mapping, path_gpkg, nom_lyr, verbose=True, progress=True): """ Args: model: model_mapping: path_gpkg: nom_lyr: verbose: progress: """ ds = DataSource(path_gpkg) lm = LayerMapping(model, ds, model_mapping, layer=nom_lyr, transform=False) lm.save(strict=True, verbose=verbose, progress=progress)
def airspace_import(path=''): from django.contrib.gis.gdal import DataSource airspace_mapping = { 'mpoly': 'POLYGON', 'ident': u'IDENT', 'name': u'NAME', 'upper_desc': u'UPPER_DESC', 'upper_val': u'UPPER_VAL', 'upper_uom': u'UPPER_UOM', 'upper_code': u'UPPER_CODE', 'lower_desc': u'LOWER_DESC', 'lower_val': u'LOWER_VAL', 'lower_uom': u'LOWER_UOM', 'lower_code': u'LOWER_CODE', 'type_code': u'TYPE_CODE', 'local_type': u'LOCAL_TYPE', 'airspace_class': u'CLASS', 'mil_code': u'MIL_CODE', 'comm_name': u'COMM_NAME', 'level': u'LEVEL', 'sector': u'SECTOR', 'onshore': u'ONSHORE', 'exclusion': u'EXCLUSION', 'wkhr_code': u'WKHR_CODE', 'wkhr_remark': u'WKHR_RMK', # None: u'DST', # no data currently # None: u'GMTOFFSET', # no data currently # None: u'SHAPE_Leng', # no data currently # None: u'SHAPE_Area', # no data currently } if os.path.exists(path): # spelunk the source ds = DataSource(path) layer = ds[0] print('Fields: %s' % layer.fields) print('Geom Type: %s' % layer.geom_type) print('SRS: %s' % layer.srs) lm = LayerMapping(Airspace, path, airspace_mapping) lm.save(verbose=True) else: raise NoShapeFileException
def read_shp(request): ds = DataSource('/opt/terra_indigena/tis_sirgas2000.shp') print ds[0].fields print ds[0][0][0] print ds[0][0].get('superficie') print ds[0][0].geom mapping = {'nome': 'terrai_nom', 'geom': 'POLYGON'} lm = LayerMapping(Indigena, '/opt/terra_indigena/tis_sirgas2000.shp', mapping) lm.save(verbose=True) parcelas = Indigena.objects.filter(nome__icontains='k') #print len(parcelas) return render_to_response('terra_indigena.html', {"geo": parcelas}, context_instance=RequestContext(request))
def run(verbose=True): lm = LayerMapping(WorldBorder, f"{world_shp}", world_mapping, transform=False) lm.save(strict=True, verbose=verbose) lm2 = LayerMapping(Campsite, f"{campsite_shp}", campsite_mapping, transform=False) lm2.save(strict=True, verbose=verbose)
def handle(self, *args, **options): dataSet = DataSource( os.path.join(os.path.dirname(__file__), 'datafiles', 'Streets_Permit_Jobs_Line.geojson')) fieldMap = { 'objectid': 'OBJECTID', 'linkpath': 'LinkPath', 'projectid': 'ProjectID', 'projectname': 'ProjectName', 'status': 'Status', 'contactname': 'ContactName', 'shape_length': 'Shape_Length', 'geom': 'UNKNOWN', } lm = LayerMapping(StPJline, dataSet, fieldMap) lm.save()
def setUp(self): # met regions dt = models.DivisionType(name='met_region') dt.save() def pre_save_callback(sender, instance, *args, **kwargs): instance.type = dt mapping = { 'name': 'NAME', 'mpoly': 'MULTIPOLYGON', } lm = LayerMapping(models.Division, METREGIONS, mapping, transform=False) pre_save.connect(pre_save_callback, sender=models.Division) try: lm.save(strict=True, verbose=False) except Exception as exc: print repr(exc) raise finally: pre_save.disconnect(pre_save_callback, sender=models.Division) # toy regions dt = models.DivisionType(name='toy') dt.save() coords = [ [(1, 1), (2, 1), (2, 2), (1, 2), (1, 1)], [(2, 1), (3, 1), (3, 2), (2, 2), (2, 1)], [(3, 1), (4, 1), (4, 2), (3, 2), (3, 1)], [(1, 2), (2, 2), (2, 3), (1, 3), (1, 2)], [(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)], [(3, 2), (4, 2), (4, 3), (3, 3), (3, 2)], [(1, 3), (2, 3), (2, 4), (1, 4), (1, 3)], [(2, 3), (3, 3), (3, 4), (2, 4), (2, 3)], [(3, 3), (4, 3), (4, 4), (3, 4), (3, 3)], [(4, 3), (5, 3), (5, 4), (4, 4), (4, 3)], ] for (i, c) in enumerate(coords): mpoly = MultiPolygon(Polygon(c)) models.Division(name="%02u" % (i+1), code=str(i), mpoly=mpoly, type=dt).save()
def run(verbose=True): lm_sp = LayerMapping(SaoPaulo, saopaulo_shp, saopaulo_mapping, transform=False, encoding='iso-8859-1') lm_mn = LayerMapping(Municipios, municipios_shp, municipios_mapping, transform=False, encoding='iso-8859-1') lm_sp.save(strict=True, verbose=True) lm_mn.save(strict=True, verbose=True)
def update_data(verbose=True): overpass_url = "http://overpass-api.de/api/interpreter" print("Sending request to ", overpass_url) response = requests.get(overpass_url, params={'data': OVERPASS_QUERY}) print("Converting OSM json to geojson") geojson = osmjson2geojson.convert(json.loads(response.text)) osm_id_list = [] for feature in geojson['features']: osm_id_list.append(['id']) # set active the ids in the list queryset = Places.objects.filter(id__in=osm_id_list) queryset.update(active=True) print("Inactivate removed entries") # set inactive the ids not in the list queryset = Places.objects.all().exclude(id__in=osm_id_list) queryset.update(active=False) print("Writing temporary geojson file") temp_path = Path(__file__).resolve().parent / 'data' / "temp.json" try: os.remove(temp_path) except FileNotFoundError: print("no temp file at: ", temp_path) print("but that is ok") geojson_file = open(temp_path, "w") geojson_file.write(json.dumps(geojson)) geojson_file.close() print("Execute LayerMapping") lm = LayerMapping(Places, temp_path.as_posix(), places_mapping, transform=False) print("Save entries to database") lm.save(strict=True, verbose=verbose)
def import_state(self, num): for chamber, mapping in {'su': upper_district_mapping, 'sl': lower_district_mapping}.items(): path = os.path.abspath(os.path.join( os.path.dirname(os.path.dirname(os.path.dirname( __file__))), 'data/')) path = os.path.join(path, "%s%s_d11.shp" % (chamber, num)) try: LayerMapping(District, path, mapping, source_srs=SOURCE_SRS, transform=True).save(strict=True) except: try: self.download_state_shapefiles(path, num) LayerMapping(District, path, mapping, source_srs=SOURCE_SRS, transform=True).save(strict=True) except: print 'error importing %s' % path
def building_footprints(): fps = SocrataCatalogItem.objects.get(title='Building Footprint') url, extension = fps.get_distribution_type_url('Shapefile') fps.download_distribution(url, extension) fps.extract_zip(fps.orig_file_loc) fps_shp = fps.get_staged_file_path(extension='shp') fps_mapping = { 'num_stories': 'numstories', 'shape_length': 'shape_leng', 'shape_area': 'shape_area', 'feature_code': 'featurecod', 'geopin': 'geopin', 'object_id': 'objectid', 'building_height': 'bldgheight', 'the_geom': 'MULTIPOLYGON' } lm = LayerMapping( BuildingFootprint, fps_shp, fps_mapping, transform=False, encoding='iso-8859-1' ) lm.save(strict=True, verbose=True)
def configure_models(instance, sender, **kwargs): """Save to postgis if there is a datastore. """ # Abort if a postgis DATABASE is not configured. if DYNAMIC_DATASTORE not in settings.DATABASES: return # Do not process if there is no table. base_file = instance.get_base_file()[0] if base_file is None or base_file.name != 'shp': return filename = base_file.file.path # Load the table in postgis and get a mapping from fields in the database # and fields in the Shapefile. mapping = file2pgtable(filename, instance.name) # Get a dynamic model with the same name as the layer. model_description, __ = ModelDescription.objects.get_or_create( name=instance.name) # Set up the fields with the postgis table generate_model(model_description, mapping, db_key=DYNAMIC_DATASTORE) # Get the new actual Django model. TheModel = model_description.get_django_model() # Use layermapping to load the layer with geodjango if not has_datastore: lm = LayerMapping(TheModel, filename, mapping, encoding=instance.charset, using=DYNAMIC_DATASTORE, transform=None) lm.save() else: post_save_layer(instance, sender, **kwargs)
def handle(self, shapefile, manipulator, *args, **options): try: manip_model = get_class(manipulator) except: raise Exception( "The %s model could not be found. \nBe sure and provide the complete description: <module name>.models.<manipulator model name>" % manipulator) ds = DataSource(shapefile) if len(ds) != 1: raise Exception( "Data source should only contain a single layer. Aborting.") layer = ds[0] if len(layer) != 1: raise Exception("Layer should containing ONLY a single feature") if not 'polygon' in layer.geom_type.name.lower(): print layer.geom_type.name raise Exception("This geometry must be a polygon") mapping = {'geometry': 'MULTIPOLYGON'} lm = LayerMapping(manip_model, shapefile, mapping) lm.save() manip_geom = manip_model.objects.order_by('-creation_date')[0] if options.get('region_name'): manip_geom.name = options.get('region_name') manip_geom.save() else: manip_geom.name = layer.name manip_geom.save() print "" print "The manipulaotr geometry, %s, has been added to the %s model with primary key = %s" % ( manip_geom.name, manipulator, manip_geom.pk) print "To switch to this geometry, you will need to run 'manage.py change_manipulator_geom %s %s'" % ( manip_geom.pk, manipulator) print ""