def test07_integer_overflow(self): "Testing that OFTReal fields, treated as OFTInteger, do not overflow." # Using *.dbf from Census 2010 TIGER Shapefile for Texas, # which has land area ('ALAND10') stored in a Real field # with no precision. ds = DataSource(os.path.join(TEST_DATA, 'texas.dbf')) feat = ds[0][0] # Reference value obtained using `ogrinfo`. self.assertEqual(676586997978, feat.get('ALAND10'))
def test03b_layer_slice(self): "Test indexing and slicing on Layers." # Using the first data-source because the same slice # can be used for both the layer and the control values. source = ds_list[0] ds = DataSource(source.ds) sl = slice(1, 3) feats = ds[0][sl] for fld_name in ds[0].fields: test_vals = [feat.get(fld_name) for feat in feats] control_vals = source.field_values[fld_name][sl] self.assertEqual(control_vals, test_vals)
def test05_geometries(self): "Testing Geometries from Data Source Features." for source in ds_list: ds = DataSource(source.ds) # Incrementing through each layer and feature. for layer in ds: for feat in layer: g = feat.geom # Making sure we get the right Geometry name & type self.assertEqual(source.geom, g.geom_name) self.assertEqual(source.gtype, g.geom_type) # Making sure the SpatialReference is as expected. if hasattr(source, 'srs_wkt'): self.assertEqual(source.srs_wkt, g.srs.wkt)
def test03_layermap_strict(self): "Testing the `strict` keyword, and import of a LineString shapefile." # When the `strict` keyword is set an error encountered will force # the importation to stop. try: lm = LayerMapping(Interstate, inter_shp, inter_mapping) lm.save(silent=True, strict=True) except InvalidDecimal: # No transactions for geoms on MySQL; delete added features. if mysql: Interstate.objects.all().delete() else: self.fail( 'Should have failed on strict import with invalid decimal values.' ) # This LayerMapping should work b/c `strict` is not set. lm = LayerMapping(Interstate, inter_shp, inter_mapping) lm.save(silent=True) # Two interstate should have imported correctly. self.assertEqual(2, Interstate.objects.count()) # Verifying the values in the layer w/the model. ds = DataSource(inter_shp) # Only the first two features of this shapefile are valid. valid_feats = ds[0][:2] for feat in valid_feats: istate = Interstate.objects.get(name=feat['Name'].value) if feat.fid == 0: self.assertEqual(Decimal(str(feat['Length'])), istate.length) elif feat.fid == 1: # Everything but the first two decimal digits were truncated, # because the Interstate model's `length` field has decimal_places=2. self.assertAlmostEqual(feat.get('Length'), float(istate.length), 2) for p1, p2 in zip(feat.geom, istate.path): self.assertAlmostEqual(p1[0], p2[0], 6) self.assertAlmostEqual(p1[1], p2[1], 6)
def test02_simple_layermap(self): "Test LayerMapping import of a simple point shapefile." # Setting up for the LayerMapping. lm = LayerMapping(City, city_shp, city_mapping) lm.save() # There should be three cities in the shape file. self.assertEqual(3, City.objects.count()) # Opening up the shapefile, and verifying the values in each # of the features made it to the model. ds = DataSource(city_shp) layer = ds[0] for feat in layer: city = City.objects.get(name=feat['Name'].value) self.assertEqual(feat['Population'].value, city.population) self.assertEqual(Decimal(str(feat['Density'])), city.density) self.assertEqual(feat['Created'].value, city.dt) # Comparing the geometries. pnt1, pnt2 = feat.geom, city.point self.assertAlmostEqual(pnt1.x, pnt2.x, 5) self.assertAlmostEqual(pnt1.y, pnt2.y, 5)
def test01_valid_shp(self): "Testing valid SHP Data Source files." for source in ds_list: # Loading up the data source ds = DataSource(source.ds) # Making sure the layer count is what's expected (only 1 layer in a SHP file) self.assertEqual(1, len(ds)) # Making sure GetName works self.assertEqual(source.ds, ds.name) # Making sure the driver name matches up self.assertEqual(source.driver, str(ds.driver)) # Making sure indexing works try: ds[len(ds)] except OGRIndexError: pass else: self.fail('Expected an IndexError!')
def test06_spatial_filter(self): "Testing the Layer.spatial_filter property." ds = DataSource(get_ds_file('cities', 'shp')) lyr = ds[0] # When not set, it should be None. self.assertEqual(None, lyr.spatial_filter) # Must be set a/an OGRGeometry or 4-tuple. self.assertRaises(TypeError, lyr._set_spatial_filter, 'foo') # Setting the spatial filter with a tuple/list with the extent of # a buffer centering around Pueblo. self.assertRaises(ValueError, lyr._set_spatial_filter, range(5)) filter_extent = (-105.609252, 37.255001, -103.609252, 39.255001) lyr.spatial_filter = (-105.609252, 37.255001, -103.609252, 39.255001) self.assertEqual(OGRGeometry.from_bbox(filter_extent), lyr.spatial_filter) feats = [feat for feat in lyr] self.assertEqual(1, len(feats)) self.assertEqual('Pueblo', feats[0].get('Name')) # Setting the spatial filter with an OGRGeometry for buffer centering # around Houston. filter_geom = OGRGeometry( 'POLYGON((-96.363151 28.763374,-94.363151 28.763374,-94.363151 30.763374,-96.363151 30.763374,-96.363151 28.763374))' ) lyr.spatial_filter = filter_geom self.assertEqual(filter_geom, lyr.spatial_filter) feats = [feat for feat in lyr] self.assertEqual(1, len(feats)) self.assertEqual('Houston', feats[0].get('Name')) # Clearing the spatial filter by setting it to None. Now # should indicate that there are 3 features in the Layer. lyr.spatial_filter = None self.assertEqual(3, len(lyr))
def mapping(data_source, geom_name='geom', layer_key=0, multi_geom=False): """ Given a DataSource, generates a dictionary that may be used for invoking the LayerMapping utility. Keyword Arguments: `geom_name` => The name of the geometry field to use for the model. `layer_key` => The key for specifying which layer in the DataSource to use; defaults to 0 (the first layer). May be an integer index or a string identifier for the layer. `multi_geom` => Boolean (default: False) - specify as multigeometry. """ if isinstance(data_source, basestring): # Instantiating the DataSource from the string. data_source = DataSource(data_source) elif isinstance(data_source, DataSource): pass else: raise TypeError( 'Data source parameter must be a string or a DataSource object.') # Creating the dictionary. _mapping = {} # Generating the field name for each field in the layer. for field in data_source[layer_key].fields: mfield = field.lower() if mfield[-1:] == '_': mfield += 'field' _mapping[mfield] = field gtype = data_source[layer_key].geom_type if multi_geom and gtype.num in (1, 2, 3): prefix = 'MULTI' else: prefix = '' _mapping[geom_name] = prefix + str(gtype).upper() return _mapping
def test04_features(self): "Testing Data Source Features." for source in ds_list: ds = DataSource(source.ds) # Incrementing through each layer for layer in ds: # Incrementing through each feature in the layer for feat in layer: # Making sure the number of fields, and the geometry type # are what's expected. self.assertEqual(source.nfld, len(list(feat))) self.assertEqual(source.gtype, feat.geom_type) # Making sure the fields match to an appropriate OFT type. for k, v in source.fields.items(): # Making sure we get the proper OGR Field instance, using # a string value index for the feature. self.assertEqual(True, isinstance(feat[k], v)) # Testing Feature.__iter__ for fld in feat: self.assertEqual(True, fld.name in source.fields.keys())
def _ogrinspect(data_source, model_name, geom_name='geom', layer_key=0, srid=None, multi_geom=False, name_field=None, imports=True, decimal=False, blank=False, null=False): """ Helper routine for `ogrinspect` that generates GeoDjango models corresponding to the given data source. See the `ogrinspect` docstring for more details. """ # Getting the DataSource if isinstance(data_source, str): data_source = DataSource(data_source) elif isinstance(data_source, DataSource): pass else: raise TypeError( 'Data source parameter must be a string or a DataSource object.') # Getting the layer corresponding to the layer key and getting # a string listing of all OGR fields in the Layer. layer = data_source[layer_key] ogr_fields = layer.fields # Creating lists from the `null`, `blank`, and `decimal` # keyword arguments. def process_kwarg(kwarg): if isinstance(kwarg, (list, tuple)): return [s.lower() for s in kwarg] elif kwarg: return [s.lower() for s in ogr_fields] else: return [] null_fields = process_kwarg(null) blank_fields = process_kwarg(blank) decimal_fields = process_kwarg(decimal) # Gets the `null` and `blank` keywords for the given field name. def get_kwargs_str(field_name): kwlist = [] if field_name.lower() in null_fields: kwlist.append('null=True') if field_name.lower() in blank_fields: kwlist.append('blank=True') if kwlist: return ', ' + ', '.join(kwlist) else: return '' # For those wishing to disable the imports. if imports: yield '# This is an auto-generated Django model module created by ogrinspect.' yield 'from my_django.contrib.gis.db import models' yield '' yield 'class %s(models.Model):' % model_name for field_name, width, precision, field_type in izip( ogr_fields, layer.field_widths, layer.field_precisions, layer.field_types): # The model field name. mfield = field_name.lower() if mfield[-1:] == '_': mfield += 'field' # Getting the keyword args string. kwargs_str = get_kwargs_str(field_name) if field_type is OFTReal: # By default OFTReals are mapped to `FloatField`, however, they # may also be mapped to `DecimalField` if specified in the # `decimal` keyword. if field_name.lower() in decimal_fields: yield ' %s = models.DecimalField(max_digits=%d, decimal_places=%d%s)' % ( mfield, width, precision, kwargs_str) else: yield ' %s = models.FloatField(%s)' % (mfield, kwargs_str[2:]) elif field_type is OFTInteger: yield ' %s = models.IntegerField(%s)' % (mfield, kwargs_str[2:]) elif field_type is OFTString: yield ' %s = models.CharField(max_length=%s%s)' % ( mfield, width, kwargs_str) elif field_type is OFTDate: yield ' %s = models.DateField(%s)' % (mfield, kwargs_str[2:]) elif field_type is OFTDateTime: yield ' %s = models.DateTimeField(%s)' % (mfield, kwargs_str[2:]) elif field_type is OFTTime: yield ' %s = models.TimeField(%s)' % (mfield, kwargs_str[2:]) else: raise TypeError('Unknown field type %s in %s' % (field_type, mfield)) # TODO: Autodetection of multigeometry types (see #7218). gtype = layer.geom_type if multi_geom and gtype.num in (1, 2, 3): geom_field = 'Multi%s' % gtype.django else: geom_field = gtype.django # Setting up the SRID keyword string. if srid is None: if layer.srs is None: srid_str = 'srid=-1' else: srid = layer.srs.srid if srid is None: srid_str = 'srid=-1' elif srid == 4326: # WGS84 is already the default. srid_str = '' else: srid_str = 'srid=%s' % srid else: srid_str = 'srid=%s' % srid yield ' %s = models.%s(%s)' % (geom_name, geom_field, srid_str) yield ' objects = models.GeoManager()' if name_field: yield '' yield ' def __unicode__(self): return self.%s' % name_field
def __init__(self, model, data, mapping, layer=0, source_srs=None, encoding=None, transaction_mode='commit_on_success', transform=True, unique=None, using=DEFAULT_DB_ALIAS): """ A LayerMapping object is initialized using the given Model (not an instance), a DataSource (or string path to an OGR-supported data file), and a mapping dictionary. See the module level docstring for more details and keyword argument usage. """ # Getting the DataSource and the associated Layer. if isinstance(data, basestring): self.ds = DataSource(data) else: self.ds = data self.layer = self.ds[layer] self.using = using self.spatial_backend = connections[using].ops # Setting the mapping & model attributes. self.mapping = mapping self.model = model # Checking the layer -- intitialization of the object will fail if # things don't check out before hand. self.check_layer() # Getting the geometry column associated with the model (an # exception will be raised if there is no geometry column). if self.spatial_backend.mysql: transform = False else: self.geo_field = self.geometry_field() # Checking the source spatial reference system, and getting # the coordinate transformation object (unless the `transform` # keyword is set to False) if transform: self.source_srs = self.check_srs(source_srs) self.transform = self.coord_transform() else: self.transform = transform # Setting the encoding for OFTString fields, if specified. if encoding: # Making sure the encoding exists, if not a LookupError # exception will be thrown. from codecs import lookup lookup(encoding) self.encoding = encoding else: self.encoding = None if unique: self.check_unique(unique) transaction_mode = 'autocommit' # Has to be set to autocommit. self.unique = unique else: self.unique = None # Setting the transaction decorator with the function in the # transaction modes dictionary. if transaction_mode in self.TRANSACTION_MODES: self.transaction_decorator = self.TRANSACTION_MODES[transaction_mode] self.transaction_mode = transaction_mode else: raise LayerMapError('Unrecognized transaction mode: %s' % transaction_mode)
def get_layer(): # This DataSource object is not accessible outside this # scope. However, a reference should still be kept alive # on the `Layer` returned. ds = DataSource(source.ds) return ds[0]
def test03a_layers(self): "Testing Data Source Layers." print "\nBEGIN - expecting out of range feature id error; safe to ignore.\n" for source in ds_list: ds = DataSource(source.ds) # Incrementing through each layer, this tests DataSource.__iter__ for layer in ds: # Making sure we get the number of features we expect self.assertEqual(len(layer), source.nfeat) # Making sure we get the number of fields we expect self.assertEqual(source.nfld, layer.num_fields) self.assertEqual(source.nfld, len(layer.fields)) # Testing the layer's extent (an Envelope), and it's properties if source.driver == 'VRT' and (GDAL_VERSION >= (1, 7, 0) and GDAL_VERSION < (1, 7, 3)): # There's a known GDAL regression with retrieving the extent # of a VRT layer in versions 1.7.0-1.7.2: # http://trac.osgeo.org/gdal/ticket/3783 pass else: self.assertEqual(True, isinstance(layer.extent, Envelope)) self.assertAlmostEqual(source.extent[0], layer.extent.min_x, 5) self.assertAlmostEqual(source.extent[1], layer.extent.min_y, 5) self.assertAlmostEqual(source.extent[2], layer.extent.max_x, 5) self.assertAlmostEqual(source.extent[3], layer.extent.max_y, 5) # Now checking the field names. flds = layer.fields for f in flds: self.assertEqual(True, f in source.fields) # Negative FIDs are not allowed. self.assertRaises(OGRIndexError, layer.__getitem__, -1) self.assertRaises(OGRIndexError, layer.__getitem__, 50000) if hasattr(source, 'field_values'): fld_names = source.field_values.keys() # Testing `Layer.get_fields` (which uses Layer.__iter__) for fld_name in fld_names: self.assertEqual(source.field_values[fld_name], layer.get_fields(fld_name)) # Testing `Layer.__getitem__`. for i, fid in enumerate(source.fids): feat = layer[fid] self.assertEqual(fid, feat.fid) # Maybe this should be in the test below, but we might as well test # the feature values here while in this loop. for fld_name in fld_names: self.assertEqual(source.field_values[fld_name][i], feat.get(fld_name)) print "\nEND - expecting out of range feature id error; safe to ignore."