class OLMap(self.widget):
template = self.map_template
geom_type = db_field.geom_type
params = {
'default_lon':
self.default_lon,
'default_lat':
self.default_lat,
'default_zoom':
self.default_zoom,
'display_wkt':
self.debug or self.display_wkt,
'geom_type':
OGRGeomType(db_field.geom_type),
'field_name':
db_field.name,
'is_collection':
is_collection,
'scrollable':
self.scrollable,
'layerswitcher':
self.layerswitcher,
'collection_type':
collection_type,
'is_linestring':
db_field.geom_type in ('LINESTRING', 'MULTILINESTRING'),
'is_polygon':
db_field.geom_type in ('POLYGON', 'MULTIPOLYGON'),
'is_point':
db_field.geom_type in ('POINT', 'MULTIPOINT'),
'num_zoom':
self.num_zoom,
'max_zoom':
self.max_zoom,
'min_zoom':
self.min_zoom,
'units':
self.units, #likely shoud get from object
'max_resolution':
self.max_resolution,
'max_extent':
self.max_extent,
'modifiable':
self.modifiable,
'mouse_position':
self.mouse_position,
'scale_text':
self.scale_text,
'map_width':
self.map_width,
'map_height':
self.map_height,
'point_zoom':
self.point_zoom,
'srid':
self.map_srid,
'display_srid':
self.display_srid,
'wms_url':
self.wms_url,
'wms_layer':
self.wms_layer,
'wms_name':
self.wms_name,
'debug':
self.debug,
}
def test00b_geomtype_25d(self):
"Testing OGRGeomType object with 25D types."
wkb25bit = OGRGeomType.wkb25bit
self.assertEqual(OGRGeomType(wkb25bit + 1), 'Point25D')
self.assertEqual(OGRGeomType('MultiLineString25D'), (5 + wkb25bit))
self.assertEqual('GeometryCollectionField', OGRGeomType('GeometryCollection25D').django)
def get_map_widget(self, db_field):
"""
Returns a subclass of the OpenLayersWidget (or whatever was specified
in the `widget` attribute) using the settings from the attributes set
in this class.
"""
is_collection = db_field.geom_type in ('MULTIPOINT', 'MULTILINESTRING',
'MULTIPOLYGON',
'GEOMETRYCOLLECTION')
if is_collection:
if db_field.geom_type == 'GEOMETRYCOLLECTION':
collection_type = 'Any'
else:
collection_type = OGRGeomType(
db_field.geom_type.replace('MULTI', ''))
else:
collection_type = 'None'
class OLMap(self.widget):
template = self.map_template
geom_type = db_field.geom_type
params = {
'default_lon':
self.default_lon,
'default_lat':
self.default_lat,
'default_zoom':
self.default_zoom,
'display_wkt':
self.debug or self.display_wkt,
'geom_type':
OGRGeomType(db_field.geom_type),
'field_name':
db_field.name,
'is_collection':
is_collection,
'scrollable':
self.scrollable,
'layerswitcher':
self.layerswitcher,
'collection_type':
collection_type,
'is_linestring':
db_field.geom_type in ('LINESTRING', 'MULTILINESTRING'),
'is_polygon':
db_field.geom_type in ('POLYGON', 'MULTIPOLYGON'),
'is_point':
db_field.geom_type in ('POINT', 'MULTIPOINT'),
'num_zoom':
self.num_zoom,
'max_zoom':
self.max_zoom,
'min_zoom':
self.min_zoom,
'units':
self.units, #likely shoud get from object
'max_resolution':
self.max_resolution,
'max_extent':
self.max_extent,
'modifiable':
self.modifiable,
'mouse_position':
self.mouse_position,
'scale_text':
self.scale_text,
'map_width':
self.map_width,
'map_height':
self.map_height,
'point_zoom':
self.point_zoom,
'srid':
self.map_srid,
'display_srid':
self.display_srid,
'wms_url':
self.wms_url,
'wms_layer':
self.wms_layer,
'wms_name':
self.wms_name,
'debug':
self.debug,
}
return OLMap
def test00_geomtype(self):
"Testing OGRGeomType object."
# OGRGeomType should initialize on all these inputs.
try:
g = OGRGeomType(1)
g = OGRGeomType(7)
g = OGRGeomType('point')
g = OGRGeomType('GeometrycollectioN')
g = OGRGeomType('LINearrING')
g = OGRGeomType('Unknown')
except:
self.fail('Could not create an OGRGeomType object!')
# Should throw TypeError on this input
self.assertRaises(TypeError, OGRGeomType.__init__, 23)
self.assertRaises(TypeError, OGRGeomType.__init__, 'fooD')
self.assertRaises(TypeError, OGRGeomType.__init__, 9)
# Equivalence can take strings, ints, and other OGRGeomTypes
self.assertEqual(True, OGRGeomType(1) == OGRGeomType(1))
self.assertEqual(True, OGRGeomType(7) == 'GeometryCollection')
self.assertEqual(True, OGRGeomType('point') == 'POINT')
self.assertEqual(False, OGRGeomType('point') == 2)
self.assertEqual(True, OGRGeomType('unknown') == 0)
self.assertEqual(True, OGRGeomType(6) == 'MULtiPolyGON')
self.assertEqual(False, OGRGeomType(1) != OGRGeomType('point'))
self.assertEqual(True, OGRGeomType('POINT') != OGRGeomType(6))
# Testing the Django field name equivalent property.
self.assertEqual('PointField', OGRGeomType('Point').django)
self.assertEqual(None, OGRGeomType('Unknown').django)
self.assertEqual(None, OGRGeomType('none').django)
def test00a_geomtype(self):
"Testing OGRGeomType object."
# OGRGeomType should initialize on all these inputs.
OGRGeomType(1)
OGRGeomType(7)
OGRGeomType('point')
OGRGeomType('GeometrycollectioN')
OGRGeomType('LINearrING')
OGRGeomType('Unknown')
# Should throw TypeError on this input
self.assertRaises(OGRException, OGRGeomType, 23)
self.assertRaises(OGRException, OGRGeomType, 'fooD')
self.assertRaises(OGRException, OGRGeomType, 9)
# Equivalence can take strings, ints, and other OGRGeomTypes
self.assertEqual(OGRGeomType(1), OGRGeomType(1))
self.assertEqual(OGRGeomType(7), 'GeometryCollection')
self.assertEqual(OGRGeomType('point'), 'POINT')
self.assertNotEqual(OGRGeomType('point'), 2)
self.assertEqual(OGRGeomType('unknown'), 0)
self.assertEqual(OGRGeomType(6), 'MULtiPolyGON')
self.assertEqual(OGRGeomType(1), OGRGeomType('point'))
self.assertNotEqual(OGRGeomType('POINT'), OGRGeomType(6))
# Testing the Django field name equivalent property.
self.assertEqual('PointField', OGRGeomType('Point').django)
self.assertEqual('GeometryField', OGRGeomType('Geometry').django)
self.assertEqual('GeometryField', OGRGeomType('Unknown').django)
self.assertIsNone(OGRGeomType('none').django)
# 'Geometry' initialization implies an unknown geometry type.
gt = OGRGeomType('Geometry')
self.assertEqual(0, gt.num)
self.assertEqual('Unknown', gt.name)
def test_geomtype(self):
"Testing OGRGeomType object."
# OGRGeomType should initialize on all these inputs.
OGRGeomType(1)
OGRGeomType(7)
OGRGeomType("point")
OGRGeomType("GeometrycollectioN")
OGRGeomType("LINearrING")
OGRGeomType("Unknown")
# Should throw TypeError on this input
with self.assertRaises(GDALException):
OGRGeomType(23)
with self.assertRaises(GDALException):
OGRGeomType("fooD")
with self.assertRaises(GDALException):
OGRGeomType(9)
# Equivalence can take strings, ints, and other OGRGeomTypes
self.assertEqual(OGRGeomType(1), OGRGeomType(1))
self.assertEqual(OGRGeomType(7), "GeometryCollection")
self.assertEqual(OGRGeomType("point"), "POINT")
self.assertNotEqual(OGRGeomType("point"), 2)
self.assertEqual(OGRGeomType("unknown"), 0)
self.assertEqual(OGRGeomType(6), "MULtiPolyGON")
self.assertEqual(OGRGeomType(1), OGRGeomType("point"))
self.assertNotEqual(OGRGeomType("POINT"), OGRGeomType(6))
# Testing the Django field name equivalent property.
self.assertEqual("PointField", OGRGeomType("Point").django)
self.assertEqual("GeometryField", OGRGeomType("Geometry").django)
self.assertEqual("GeometryField", OGRGeomType("Unknown").django)
self.assertIsNone(OGRGeomType("none").django)
# 'Geometry' initialization implies an unknown geometry type.
gt = OGRGeomType("Geometry")
self.assertEqual(0, gt.num)
self.assertEqual("Unknown", gt.name)
def get_geometry_type(self, table_name, geo_col):
"""
The geometry type used by SQL Server doesn't contain any
specific type, SRID,or dimension information; that is
associated with instances. This makes introspection hacky and
error prone, but presumably that also applies to the data
integrity.
"""
cursor = self.connection.cursor()
try:
field_type, field_params = None, {}
# Look for a sample piece of data and assume that it's
# representative. If we can't find anything, we'll have
# to examine the constraints:
try:
cursor.excute("SELECT TOP 1 [%(geo_col)].STGeometryType(), " +
"[%(geo_col)].STDimension(), " +
"[%(geo_col)].STSrid " +
"FROM [%(table_name)] " +
"WHERE [%(geo_col)] IS NOT NULL" % locals())
row = cursor.fetchone()
if not rows:
raise GeoIntrospectionError
field_type = OGRGeomType(row[0]).django
srid, dim = row[1:]
if srid != 4326:
field_params['srid'] = srid
if dim != 2:
field_params['dim'] = dim
except GeoIntrospectionError:
# The table is empty, so we'll have to guess based on
# parsing the constraint information. Unfortunately,
# this will only give us the type, but not dimension
# or SRID (and nothing reliably)
cursor.execute(
"SELECT [CHECK_CLAUSE]"
"FROM [INFORMATION_SCHEMA].[CHECK_CONSTRAINTS] cc"
"JOIN [INFORMATION_SCHEMA].[CONSTRAINT_COLUMN_USAGE] ccu"
"ON ccu.[CONSTRAINT_NAME] = cc.[CONSTRAINT_NAME]"
"WHERE ccu.[CONSTRAINT_CATALOG] = '%s'"
"AND [COLUMN_NAME] = '%s'" % (table_name, geo_col))
rows = cursor.fetchall()
if rows:
# Sample to parse:
# ([geom].[STGeometryType]()='Point')
type_re = re.compile(r"\[STGeometryType\]\(\)='(\w+)'")
for row in rows:
constraint = row[0]
m = type_re.search(constraint)
if m:
field_type = OGRGeomType(m.group(1)).django
break
finally:
cursor.close()
return field_type, field_params
def convert_shapefile(shapefilename, srid=4674):
"""
shapefilename: considera nomenclatura de shapefile do IBGE para determinar se é UF
ou Municípios.
ex. 55UF2500GC_SIR.shp para UF e 55MU2500GC_SIR.shp para Municípios
srid: 4674 (Projeção SIRGAS 2000)
"""
# /home/nando/Desktop/IBGE/2010/55MU2500GC_SIR.shp
ds = DataSource(shapefilename)
is_uf = shapefilename.upper().find('UF') != -1
transform_coord = None
if srid != SRID:
transform_coord = CoordTransform(SpatialReference(srid),
SpatialReference(SRID))
if is_uf:
model = UF
else:
model = Municipio
ct = 0
for f in ds[0]:
# 3D para 2D se necessário
if f.geom.coord_dim != 2:
f.geom.coord_dim = 2
# converte para MultiPolygon se necessário
if isinstance(f.geom, Polygon):
g = OGRGeometry(OGRGeomType('MultiPolygon'))
g.add(f.geom)
else:
g = f.geom
# transforma coordenadas se necessário
if transform_coord:
g.transform(transform_coord)
# força 2D
g.coord_dim = 2
kwargs = {}
if is_uf:
kwargs['nome'] = capitalize_name(
unicode(f.get(CAMPO_NOME_UF), 'latin1'))
kwargs['geom'] = g.ewkt
kwargs['id_ibge'] = f.get(CAMPO_GEOCODIGO_UF)
kwargs['regiao'] = capitalize_name(
unicode(f.get(CAMPO_REGIAO_UF), 'latin1'))
kwargs['uf'] = UF_SIGLAS_DICT.get(kwargs['id_ibge'])
else:
kwargs['nome'] = capitalize_name(
unicode(f.get(CAMPO_NOME_MU), 'latin1'))
kwargs['geom'] = g.ewkt
kwargs['id_ibge'] = f.get(CAMPO_GEOCODIGO_MU)
kwargs['uf'] = UF.objects.get(pk=f.get(CAMPO_GEOCODIGO_MU)[:2])
kwargs['uf_sigla'] = kwargs['uf'].uf
kwargs['nome_abreviado'] = slugify(kwargs['nome'])
# tenta corrigir nomes duplicados, são em torno de 242 nomes repetidos
# adicionando a sigla do estado no final
if Municipio.objects.filter(
nome_abreviado=kwargs['nome_abreviado']).count() > 0:
kwargs['nome_abreviado'] = u'%s-%s' % (
kwargs['nome_abreviado'], kwargs['uf_sigla'].lower())
instance = model(**kwargs)
instance.save()
ct += 1
print ct, (is_uf and "Unidades Federativas criadas"
or "Municipios criados")
def write_with_native_bindings(self, tmp_name, queryset, geo_field):
"""Scrive un file in un formato geografico da un geoqueryset; questa
funzione usa le librerie Python di GDAL.
Scritto da Jared Kibele e Dane Springmeyer.
"""
dr = ogr.GetDriverByName(str(self.out_format))
ds = dr.CreateDataSource(tmp_name)
if ds is None:
raise Exception('Could not create file!')
if hasattr(geo_field, 'geom_type'):
ogr_type = OGRGeomType(geo_field.geom_type).num
else:
ogr_type = OGRGeomType(geo_field._geom).num
native_srs = osr.SpatialReference()
if hasattr(geo_field, 'srid'):
native_srs.ImportFromEPSG(geo_field.srid)
else:
native_srs.ImportFromEPSG(geo_field._srid)
if self.proj_transform:
output_srs = osr.SpatialReference()
output_srs.ImportFromEPSG(self.proj_transform)
else:
output_srs = native_srs
layer = ds.CreateLayer('lyr', srs=output_srs, geom_type=ogr_type)
attributes = self.get_attributes()
for field in attributes:
field_defn = ogr.FieldDefn(str(field.name), ogr.OFTString)
field_defn.SetWidth(255)
if layer.CreateField(field_defn) != 0:
raise Exception('Faild to create field')
feature_def = layer.GetLayerDefn()
for item in queryset:
feat = ogr.Feature(feature_def)
for field in attributes:
value = getattr(item, field.name)
try:
string_value = str(value)
except UnicodeEncodeError, E:
string_value = ''
feat.SetField(str(field.name), string_value)
geom = getattr(item, geo_field.name)
if geom:
ogr_geom = ogr.CreateGeometryFromWkt(geom.wkt)
if self.proj_transform:
ct = osr.CoordinateTransformation(native_srs, output_srs)
ogr_geom.Transform(ct)
check_err(feat.SetGeometry(ogr_geom))
else:
pass
check_err(layer.CreateFeature(feat))
raise Exception('Could not find a geometry column for "%s"."%s"' %
(table_name, geo_col))
# OGRGeomType does not require GDAL and makes it easy to convert
# from OGC geom type name to Django field.
ogr_type = row[2]
<<<<<<< HEAD
if isinstance(ogr_type, six.integer_types) and ogr_type > 1000:
=======
if isinstance(ogr_type, int) and ogr_type > 1000:
>>>>>>> 37c99181c9a6b95433d60f8c8ef9af5731096435
# SpatiaLite versions >= 4 use the new SFSQL 1.2 offsets
# 1000 (Z), 2000 (M), and 3000 (ZM) to indicate the presence of
# higher dimensional coordinates (M not yet supported by Django).
ogr_type = ogr_type % 1000 + OGRGeomType.wkb25bit
field_type = OGRGeomType(ogr_type).django
# Getting any GeometryField keyword arguments that are not the default.
dim = row[0]
srid = row[1]
field_params = {}
if srid != 4326:
field_params['srid'] = srid
<<<<<<< HEAD
if (isinstance(dim, six.string_types) and 'Z' in dim) or dim == 3:
=======
if (isinstance(dim, str) and 'Z' in dim) or dim == 3:
>>>>>>> 37c99181c9a6b95433d60f8c8ef9af5731096435
field_params['dim'] = 3
finally:
cursor.close()
class LayerMapping(object):
"A class that maps OGR Layers to GeoDjango Models."
# Acceptable 'base' types for a multi-geometry type.
MULTI_TYPES = {
1: OGRGeomType('MultiPoint'),
2: OGRGeomType('MultiLineString'),
3: OGRGeomType('MultiPolygon'),
OGRGeomType('Point25D').num: OGRGeomType('MultiPoint25D'),
OGRGeomType('LineString25D').num: OGRGeomType('MultiLineString25D'),
OGRGeomType('Polygon25D').num: OGRGeomType('MultiPolygon25D'),
}
# Acceptable Django field types and corresponding acceptable OGR
# counterparts.
FIELD_TYPES = {
models.AutoField:
OFTInteger,
models.IntegerField: (OFTInteger, OFTReal, OFTString),
models.FloatField: (OFTInteger, OFTReal),
models.DateField:
OFTDate,
models.DateTimeField:
OFTDateTime,
models.EmailField:
OFTString,
models.TimeField:
OFTTime,
models.DecimalField: (OFTInteger, OFTReal),
models.CharField:
OFTString,
models.SlugField:
OFTString,
models.TextField:
OFTString,
models.URLField:
OFTString,
USStateField:
OFTString,
# This is a reminder that XMLField is deprecated
# and this needs to be removed in 1.4
models.XMLField:
OFTString,
models.SmallIntegerField: (OFTInteger, OFTReal, OFTString),
models.PositiveSmallIntegerField: (OFTInteger, OFTReal, OFTString),
}
# The acceptable transaction modes.
TRANSACTION_MODES = {
'autocommit': transaction.autocommit,
'commit_on_success': transaction.commit_on_success,
}
def __init__(self,
model,
data,
mapping,
layer=0,
source_srs=None,
encoding=None,
transaction_mode='commit_on_success',
transform=True,
unique=None,
using=DEFAULT_DB_ALIAS):
"""
A LayerMapping object is initialized using the given Model (not an instance),
a DataSource (or string path to an OGR-supported data file), and a mapping
dictionary. See the module level docstring for more details and keyword
argument usage.
"""
# Getting the DataSource and the associated Layer.
if isinstance(data, basestring):
self.ds = DataSource(data)
else:
self.ds = data
self.layer = self.ds[layer]
self.using = using
self.spatial_backend = connections[using].ops
# Setting the mapping & model attributes.
self.mapping = mapping
self.model = model
# Checking the layer -- intitialization of the object will fail if
# things don't check out before hand.
self.check_layer()
# Getting the geometry column associated with the model (an
# exception will be raised if there is no geometry column).
if self.spatial_backend.mysql:
transform = False
else:
self.geo_field = self.geometry_field()
# Checking the source spatial reference system, and getting
# the coordinate transformation object (unless the `transform`
# keyword is set to False)
if transform:
self.source_srs = self.check_srs(source_srs)
self.transform = self.coord_transform()
else:
self.transform = transform
# Setting the encoding for OFTString fields, if specified.
if encoding:
# Making sure the encoding exists, if not a LookupError
# exception will be thrown.
from codecs import lookup
lookup(encoding)
self.encoding = encoding
else:
self.encoding = None
if unique:
self.check_unique(unique)
transaction_mode = 'autocommit' # Has to be set to autocommit.
self.unique = unique
else:
self.unique = None
# Setting the transaction decorator with the function in the
# transaction modes dictionary.
if transaction_mode in self.TRANSACTION_MODES:
self.transaction_decorator = self.TRANSACTION_MODES[
transaction_mode]
self.transaction_mode = transaction_mode
else:
raise LayerMapError('Unrecognized transaction mode: %s' %
transaction_mode)
if using is None:
pass
#### Checking routines used during initialization ####
def check_fid_range(self, fid_range):
"This checks the `fid_range` keyword."
if fid_range:
if isinstance(fid_range, (tuple, list)):
return slice(*fid_range)
elif isinstance(fid_range, slice):
return fid_range
else:
raise TypeError
else:
return None
def check_layer(self):
"""
This checks the Layer metadata, and ensures that it is compatible
with the mapping information and model. Unlike previous revisions,
there is no need to increment through each feature in the Layer.
"""
# The geometry field of the model is set here.
# TODO: Support more than one geometry field / model. However, this
# depends on the GDAL Driver in use.
self.geom_field = False
self.fields = {}
# Getting lists of the field names and the field types available in
# the OGR Layer.
ogr_fields = self.layer.fields
ogr_field_types = self.layer.field_types
# Function for determining if the OGR mapping field is in the Layer.
def check_ogr_fld(ogr_map_fld):
try:
idx = ogr_fields.index(ogr_map_fld)
except ValueError:
raise LayerMapError(
'Given mapping OGR field "%s" not found in OGR Layer.' %
ogr_map_fld)
return idx
# No need to increment through each feature in the model, simply check
# the Layer metadata against what was given in the mapping dictionary.
for field_name, ogr_name in self.mapping.items():
# Ensuring that a corresponding field exists in the model
# for the given field name in the mapping.
try:
model_field = self.model._meta.get_field(field_name)
except models.fields.FieldDoesNotExist:
raise LayerMapError(
'Given mapping field "%s" not in given Model fields.' %
field_name)
# Getting the string name for the Django field class (e.g., 'PointField').
fld_name = model_field.__class__.__name__
if isinstance(model_field, GeometryField):
if self.geom_field:
raise LayerMapError(
'LayerMapping does not support more than one GeometryField per model.'
)
# Getting the coordinate dimension of the geometry field.
coord_dim = model_field.dim
try:
if coord_dim == 3:
gtype = OGRGeomType(ogr_name + '25D')
else:
gtype = OGRGeomType(ogr_name)
except OGRException:
raise LayerMapError(
'Invalid mapping for GeometryField "%s".' % field_name)
# Making sure that the OGR Layer's Geometry is compatible.
ltype = self.layer.geom_type
if not (ltype.name.startswith(gtype.name)
or self.make_multi(ltype, model_field)):
raise LayerMapError(
'Invalid mapping geometry; model has %s%s, '
'layer geometry type is %s.' %
(fld_name,
(coord_dim == 3 and '(dim=3)') or '', ltype))
# Setting the `geom_field` attribute w/the name of the model field
# that is a Geometry. Also setting the coordinate dimension
# attribute.
self.geom_field = field_name
self.coord_dim = coord_dim
fields_val = model_field
elif isinstance(model_field, models.ForeignKey):
if isinstance(ogr_name, dict):
# Is every given related model mapping field in the Layer?
rel_model = model_field.rel.to
for rel_name, ogr_field in ogr_name.items():
idx = check_ogr_fld(ogr_field)
try:
rel_field = rel_model._meta.get_field(rel_name)
except models.fields.FieldDoesNotExist:
raise LayerMapError(
'ForeignKey mapping field "%s" not in %s fields.'
% (rel_name, rel_model.__class__.__name__))
fields_val = rel_model
else:
raise TypeError(
'ForeignKey mapping must be of dictionary type.')
else:
# Is the model field type supported by LayerMapping?
if not model_field.__class__ in self.FIELD_TYPES:
raise LayerMapError(
'Django field type "%s" has no OGR mapping (yet).' %
fld_name)
# Is the OGR field in the Layer?
idx = check_ogr_fld(ogr_name)
ogr_field = ogr_field_types[idx]
# Can the OGR field type be mapped to the Django field type?
if not issubclass(ogr_field,
self.FIELD_TYPES[model_field.__class__]):
raise LayerMapError(
'OGR field "%s" (of type %s) cannot be mapped to Django %s.'
% (ogr_field, ogr_field.__name__, fld_name))
fields_val = model_field
self.fields[field_name] = fields_val
def check_srs(self, source_srs):
"Checks the compatibility of the given spatial reference object."
if isinstance(source_srs, SpatialReference):
sr = source_srs
elif isinstance(source_srs, self.spatial_backend.spatial_ref_sys()):
sr = source_srs.srs
elif isinstance(source_srs, (int, basestring)):
sr = SpatialReference(source_srs)
else:
# Otherwise just pulling the SpatialReference from the layer
sr = self.layer.srs
if not sr:
raise LayerMapError('No source reference system defined.')
else:
return sr
def check_unique(self, unique):
"Checks the `unique` keyword parameter -- may be a sequence or string."
if isinstance(unique, (list, tuple)):
# List of fields to determine uniqueness with
for attr in unique:
if not attr in self.mapping: raise ValueError
elif isinstance(unique, basestring):
# Only a single field passed in.
if unique not in self.mapping: raise ValueError
else:
raise TypeError(
'Unique keyword argument must be set with a tuple, list, or string.'
)
#### Keyword argument retrieval routines ####
def feature_kwargs(self, feat):
"""
Given an OGR Feature, this will return a dictionary of keyword arguments
for constructing the mapped model.
"""
# The keyword arguments for model construction.
kwargs = {}
# Incrementing through each model field and OGR field in the
# dictionary mapping.
for field_name, ogr_name in self.mapping.items():
model_field = self.fields[field_name]
if isinstance(model_field, GeometryField):
# Verify OGR geometry.
try:
val = self.verify_geom(feat.geom, model_field)
except OGRException:
raise LayerMapError(
'Could not retrieve geometry from feature.')
elif isinstance(model_field, models.base.ModelBase):
# The related _model_, not a field was passed in -- indicating
# another mapping for the related Model.
val = self.verify_fk(feat, model_field, ogr_name)
else:
# Otherwise, verify OGR Field type.
val = self.verify_ogr_field(feat[ogr_name], model_field)
# Setting the keyword arguments for the field name with the
# value obtained above.
kwargs[field_name] = val
return kwargs
def unique_kwargs(self, kwargs):
"""
Given the feature keyword arguments (from `feature_kwargs`) this routine
will construct and return the uniqueness keyword arguments -- a subset
of the feature kwargs.
"""
if isinstance(self.unique, basestring):
return {self.unique: kwargs[self.unique]}
else:
return dict((fld, kwargs[fld]) for fld in self.unique)
#### Verification routines used in constructing model keyword arguments. ####
def verify_ogr_field(self, ogr_field, model_field):
"""
Verifies if the OGR Field contents are acceptable to the Django
model field. If they are, the verified value is returned,
otherwise the proper exception is raised.
"""
if (isinstance(ogr_field, OFTString)
and isinstance(model_field,
(models.CharField, models.TextField))):
if self.encoding:
# The encoding for OGR data sources may be specified here
# (e.g., 'cp437' for Census Bureau boundary files).
val = unicode(ogr_field.value, self.encoding)
else:
val = ogr_field.value
if len(val) > model_field.max_length:
raise InvalidString(
'%s model field maximum string length is %s, given %s characters.'
% (model_field.name, model_field.max_length, len(val)))
elif isinstance(ogr_field, OFTReal) and isinstance(
model_field, models.DecimalField):
try:
# Creating an instance of the Decimal value to use.
d = Decimal(str(ogr_field.value))
except:
raise InvalidDecimal('Could not construct decimal from: %s' %
ogr_field.value)
# Getting the decimal value as a tuple.
dtup = d.as_tuple()
digits = dtup[1]
d_idx = dtup[2] # index where the decimal is
# Maximum amount of precision, or digits to the left of the decimal.
max_prec = model_field.max_digits - model_field.decimal_places
# Getting the digits to the left of the decimal place for the
# given decimal.
if d_idx < 0:
n_prec = len(digits[:d_idx])
else:
n_prec = len(digits) + d_idx
# If we have more than the maximum digits allowed, then throw an
# InvalidDecimal exception.
if n_prec > max_prec:
raise InvalidDecimal(
'A DecimalField with max_digits %d, decimal_places %d must round to an absolute value less than 10^%d.'
% (model_field.max_digits, model_field.decimal_places,
max_prec))
val = d
elif isinstance(ogr_field, (OFTReal, OFTString)) and isinstance(
model_field, models.IntegerField):
# Attempt to convert any OFTReal and OFTString value to an OFTInteger.
try:
val = int(ogr_field.value)
except:
raise InvalidInteger('Could not construct integer from: %s' %
ogr_field.value)
else:
val = ogr_field.value
return val
def verify_fk(self, feat, rel_model, rel_mapping):
"""
Given an OGR Feature, the related model and its dictionary mapping,
this routine will retrieve the related model for the ForeignKey
mapping.
"""
# TODO: It is expensive to retrieve a model for every record --
# explore if an efficient mechanism exists for caching related
# ForeignKey models.
# Constructing and verifying the related model keyword arguments.
fk_kwargs = {}
for field_name, ogr_name in rel_mapping.items():
fk_kwargs[field_name] = self.verify_ogr_field(
feat[ogr_name], rel_model._meta.get_field(field_name))
# Attempting to retrieve and return the related model.
try:
return rel_model.objects.get(**fk_kwargs)
except ObjectDoesNotExist:
raise MissingForeignKey(
'No ForeignKey %s model found with keyword arguments: %s' %
(rel_model.__name__, fk_kwargs))
def verify_geom(self, geom, model_field):
"""
Verifies the geometry -- will construct and return a GeometryCollection
if necessary (for example if the model field is MultiPolygonField while
the mapped shapefile only contains Polygons).
"""
# Downgrade a 3D geom to a 2D one, if necessary.
if self.coord_dim != geom.coord_dim:
geom.coord_dim = self.coord_dim
if self.make_multi(geom.geom_type, model_field):
# Constructing a multi-geometry type to contain the single geometry
multi_type = self.MULTI_TYPES[geom.geom_type.num]
g = OGRGeometry(multi_type)
g.add(geom)
else:
g = geom
# Transforming the geometry with our Coordinate Transformation object,
# but only if the class variable `transform` is set w/a CoordTransform
# object.
if self.transform: g.transform(self.transform)
# Returning the WKT of the geometry.
return g.wkt
#### Other model methods ####
def coord_transform(self):
"Returns the coordinate transformation object."
SpatialRefSys = self.spatial_backend.spatial_ref_sys()
try:
# Getting the target spatial reference system
target_srs = SpatialRefSys.objects.get(
srid=self.geo_field.srid).srs
# Creating the CoordTransform object
return CoordTransform(self.source_srs, target_srs)
except Exception, msg:
raise LayerMapError(
'Could not translate between the data source and model geometry: %s'
% msg)
def __call__(self):
"""
"""
fields = self.queryset.model._meta.fields
geo_fields = [f for f in fields if isinstance(f, GeometryField)]
geo_fields_names = ', '.join([f.name for f in geo_fields])
attributes = [f for f in fields if not isinstance(f, GeometryField)]
if len(geo_fields) > 1:
if not self.geo_field:
raise ValueError("More than one geodjango geometry field found, please specify which to use by name. Available fields are: '%s'" % geo_fields_names)
else:
geo_field_by_name = [fld for fld in geo_fields if fld.name == self.geo_field]
if not geo_field_by_name:
raise ValueError("Geodjango geometry field not found with the name '%s', fields available are: '%s'" % (self.geo_field,geo_fields_names))
else:
geo_field = geo_field_by_name[0]
elif geo_fields:
geo_field = geo_fields[0]
else:
raise ValueError('No geodjango geometry fields found in this model queryset')
ogr.OGRRegisterAll()
# Get the shapefile driver
dr = ogr.OGRGetDriverByName('ESRI Shapefile')
# create a temporary file to write the shapefile to
# since we are ultimately going to zip it up
tmp = tempfile.NamedTemporaryFile(suffix='.shp', mode = 'w+b')
# we must close the file for GDAL to be able to open and write to it
tmp.close()
# Creating the datasource
ds = ogr.OGR_Dr_CreateDataSource(dr,tmp.name, None)
# Get the right geometry type number for ogr
ogr_type = OGRGeomType(geo_field._geom).num
# Set up the spatial reference with epsg code
srs = SpatialReference(geo_field._srid)
# If true we're going to reproject later on
if self.proj_transform:
srs = SpatialReference(self.proj_transform)
# Creating the layer
layer = ogr.OGR_DS_CreateLayer(ds,os.path.basename(tmp.name), srs._ptr, ogr_type, None)
# Create the fields
# Todo: control field order as param
for field in attributes:
fld = ogr.OGR_Fld_Create(str(field.name), 4)
added = ogr.OGR_L_CreateField(layer, fld, 0)
check_err(added)
# Getting the Layer feature definition.
feature_def = ogr.OGR_L_GetLayerDefn(layer)
# Loop through queryset creating features
for item in self.queryset:
feat = ogr.OGR_F_Create(feature_def)
# For now, set all fields as strings
# TODO: catch model types and convert to ogr fields
# http://www.gdal.org/ogr/classOGRFeature.html
# OGR_F_SetFieldDouble
#OFTReal => FloatField DecimalField
# OGR_F_SetFieldInteger
#OFTInteger => IntegerField
#OGR_F_SetFieldStrin
#OFTString => CharField
# OGR_F_SetFieldDateTime()
#OFTDateTime => DateTimeField
#OFTDate => TimeField
#OFTDate => DateField
idx = 0
for field in attributes:
value = getattr(item,field.name)
try:
string_value = str(value)
except UnicodeEncodeError, E:
# pass for now....
# http://trac.osgeo.org/gdal/ticket/882
string_value = ''
ogr.OGR_F_SetFieldString(feat, idx, string_value)
idx += 1
# Transforming & setting the geometry
geom = getattr(item,geo_field.name)
#import pdb;pdb.set_trace()
# if requested we transform the input geometry
# to match the shapefiles projection 'to-be'
if geom:
if self.proj_transform:
geom.transform(self.proj_transform)
ogr_geom = OGRGeometry(geom.wkt,srs)
# create the geometry
check_err(ogr.OGR_F_SetGeometry(feat, ogr_geom._ptr))
else:
# Case where geometry object is not found because of null value for field
# effectively looses whole record in shapefile if geometry does not exist
pass
# creat the feature in the layer.
check_err(ogr.OGR_L_SetFeature(layer, feat))
from django.contrib.gis.gdal import OGRGeomType, OGRGeometry
MULTI_TYPES = {
1: OGRGeomType('MultiPoint'),
2: OGRGeomType('MultiLineString'),
3: OGRGeomType('MultiPolygon'),
OGRGeomType('Point25D').num: OGRGeomType('MultiPoint25D'),
OGRGeomType('LineString25D').num: OGRGeomType('MultiLineString25D'),
OGRGeomType('Polygon25D').num: OGRGeomType('MultiPolygon25D'),
}
def make_multi(geom_type, model_field):
"""
Given the OGRGeomType for a geometry and its associated GeometryField,
determine whether the geometry should be turned into a GeometryCollection.
"""
return (geom_type.num in MULTI_TYPES
and model_field.__class__.__name__ == 'Multi%s' % geom_type.django)
def verify_geom(geom, model_field):
"""
Verifies the geometry -- will construct and return a GeometryCollection
if necessary (for example if the model field is MultiPolygonField while
the mapped shapefile only contains Polygons).
"""
if make_multi(geom.geom_type, model_field):
# Constructing a multi-geometry type to contain the single geometry
multi_type = MULTI_TYPES[geom.geom_type.num]
def write_with_native_bindings(self, tmp_name, queryset, geo_field):
""" Write a shapefile out to a file from a geoqueryset.
Written by Jared Kibele and Dane Springmeyer.
In this case we use the python bindings available with a build
of gdal when compiled with --with-python, instead of the ctypes-based
bindings that GeoDjango provides.
"""
dr = ogr.GetDriverByName('ESRI Shapefile')
ds = dr.CreateDataSource(tmp_name)
if ds is None:
raise Exception('Could not create file!')
if hasattr(geo_field, 'geom_type'):
ogr_type = OGRGeomType(geo_field.geom_type).num
else:
ogr_type = OGRGeomType(geo_field._geom).num
native_srs = osr.SpatialReference()
if hasattr(geo_field, 'srid'):
native_srs.ImportFromEPSG(geo_field.srid)
else:
native_srs.ImportFromEPSG(geo_field._srid)
if self.proj_transform:
output_srs = osr.SpatialReference()
output_srs.ImportFromEPSG(self.proj_transform)
else:
output_srs = native_srs
layer = ds.CreateLayer('lyr', srs=output_srs, geom_type=ogr_type)
attributes = self.get_attributes()
for field in attributes:
# map django field type to OGR type
field_defn = self.map_ogr_types(field)
if layer.CreateField(field_defn) != 0:
raise Exception('Failed to create field')
feature_def = layer.GetLayerDefn()
for item in queryset:
feat = ogr.Feature(feature_def)
for field in attributes:
# if the field is a foreign key, return its pk value. this is
# a problem when a model is given a __unicode__ representation.
value = getattr(item, field.name)
if value is not None and isinstance(field, ForeignKey):
value = getattr(value, 'pk')
if isinstance(value, unicode):
value = value.encode('utf-8')
# truncate the field name.
# TODO: handle nonunique truncated field names.
feat.SetField(str(field.name[0:10]), value)
geom = getattr(item, geo_field.name)
if geom:
ogr_geom = ogr.CreateGeometryFromWkt(geom.wkt)
if self.proj_transform:
ct = osr.CoordinateTransformation(native_srs, output_srs)
ogr_geom.Transform(ct)
check_err(feat.SetGeometry(ogr_geom))
else:
pass
check_err(layer.CreateFeature(feat))
ds.Destroy()
def check_layer(self):
"""
Check the Layer metadata and ensure that it's compatible with the
mapping information and model. Unlike previous revisions, there is no
need to increment through each feature in the Layer.
"""
# The geometry field of the model is set here.
# TODO: Support more than one geometry field / model. However, this
# depends on the GDAL Driver in use.
self.geom_field = False
self.fields = {}
# Getting lists of the field names and the field types available in
# the OGR Layer.
ogr_fields = self.layer.fields
ogr_field_types = self.layer.field_types
# Function for determining if the OGR mapping field is in the Layer.
def check_ogr_fld(ogr_map_fld):
try:
idx = ogr_fields.index(ogr_map_fld)
except ValueError:
raise LayerMapError('Given mapping OGR field "%s" not found in OGR Layer.' % ogr_map_fld)
return idx
# No need to increment through each feature in the model, simply check
# the Layer metadata against what was given in the mapping dictionary.
for field_name, ogr_name in self.mapping.items():
# Ensuring that a corresponding field exists in the model
# for the given field name in the mapping.
try:
model_field = self.model._meta.get_field(field_name)
except FieldDoesNotExist:
raise LayerMapError('Given mapping field "%s" not in given Model fields.' % field_name)
# Getting the string name for the Django field class (e.g., 'PointField').
fld_name = model_field.__class__.__name__
if isinstance(model_field, GeometryField):
if self.geom_field:
raise LayerMapError('LayerMapping does not support more than one GeometryField per model.')
# Getting the coordinate dimension of the geometry field.
coord_dim = model_field.dim
try:
if coord_dim == 3:
gtype = OGRGeomType(ogr_name + '25D')
else:
gtype = OGRGeomType(ogr_name)
except GDALException:
raise LayerMapError('Invalid mapping for GeometryField "%s".' % field_name)
# Making sure that the OGR Layer's Geometry is compatible.
ltype = self.layer.geom_type
if not (ltype.name.startswith(gtype.name) or self.make_multi(ltype, model_field)):
raise LayerMapError('Invalid mapping geometry; model has %s%s, '
'layer geometry type is %s.' %
(fld_name, '(dim=3)' if coord_dim == 3 else '', ltype))
# Setting the `geom_field` attribute w/the name of the model field
# that is a Geometry. Also setting the coordinate dimension
# attribute.
self.geom_field = field_name
self.coord_dim = coord_dim
fields_val = model_field
elif isinstance(model_field, models.ForeignKey):
if isinstance(ogr_name, dict):
# Is every given related model mapping field in the Layer?
rel_model = model_field.remote_field.model
for rel_name, ogr_field in ogr_name.items():
idx = check_ogr_fld(ogr_field)
try:
rel_model._meta.get_field(rel_name)
except FieldDoesNotExist:
raise LayerMapError('ForeignKey mapping field "%s" not in %s fields.' %
(rel_name, rel_model.__class__.__name__))
fields_val = rel_model
else:
raise TypeError('ForeignKey mapping must be of dictionary type.')
else:
# Is the model field type supported by LayerMapping?
if model_field.__class__ not in self.FIELD_TYPES:
raise LayerMapError('Django field type "%s" has no OGR mapping (yet).' % fld_name)
# Is the OGR field in the Layer?
idx = check_ogr_fld(ogr_name)
ogr_field = ogr_field_types[idx]
# Can the OGR field type be mapped to the Django field type?
if not issubclass(ogr_field, self.FIELD_TYPES[model_field.__class__]):
raise LayerMapError('OGR field "%s" (of type %s) cannot be mapped to Django %s.' %
(ogr_field, ogr_field.__name__, fld_name))
fields_val = model_field
self.fields[field_name] = fields_val
def write_with_ctypes_bindings(self, tmp_name, queryset, geo_field):
""" Scrive un file in un formato geografico da un geoqueryset; questa
funzione usa le librerie Python di GeoDjangos.
"""
# Get the shapefile driver
dr = Driver(self.out_format)
# Creating the datasource
ds = lgdal.OGR_Dr_CreateDataSource(dr._ptr, tmp_name, None)
if ds is None:
raise Exception('Could not create file!')
# Get the right geometry type number for ogr
if hasattr(geo_field, 'geom_type'):
ogr_type = OGRGeomType(geo_field.geom_type).num
else:
ogr_type = OGRGeomType(geo_field._geom).num
# Set up the native spatial reference of geometry field using the srid
if hasattr(geo_field, 'srid'):
native_srs = SpatialReference(geo_field.srid)
else:
native_srs = SpatialReference(geo_field._srid)
if self.proj_transform:
output_srs = SpatialReference(self.proj_transform)
else:
output_srs = native_srs
# create the layer
# this is crashing python26 on osx and ubuntu
layer = lgdal.OGR_DS_CreateLayer(ds, 'lyr', output_srs._ptr, ogr_type,
None)
# Create the fields
attributes = self.get_attributes()
for field in attributes:
fld = lgdal.OGR_Fld_Create(str(field.name), 4)
added = lgdal.OGR_L_CreateField(layer, fld, 0)
check_err(added)
# Getting the Layer feature definition.
feature_def = lgdal.OGR_L_GetLayerDefn(layer)
# Loop through queryset creating features
for item in self.queryset:
feat = lgdal.OGR_F_Create(feature_def)
# For now, set all fields as strings
# TODO: catch model types and convert to ogr fields
# http://www.gdal.org/ogr/classOGRFeature.html
# OGR_F_SetFieldDouble
#OFTReal => FloatField DecimalField
# OGR_F_SetFieldInteger
#OFTInteger => IntegerField
#OGR_F_SetFieldStrin
#OFTString => CharField
# OGR_F_SetFieldDateTime()
#OFTDateTime => DateTimeField
#OFTDate => TimeField
#OFTDate => DateField
for idx, field in enumerate(attributes):
value = getattr(item, field.name)
try:
string_value = str(value)
except UnicodeEncodeError, E:
# pass for now....
# http://trac.osgeo.org/gdal/ticket/882
string_value = ''
lgdal.OGR_F_SetFieldString(feat, idx, string_value)
# Transforming & setting the geometry
geom = getattr(item, geo_field.name)
# if requested we transform the input geometry
# to match the shapefiles projection 'to-be'
if geom:
ogr_geom = OGRGeometry(geom.wkt, output_srs)
if self.proj_transform:
ct = CoordTransform(native_srs, output_srs)
ogr_geom.transform(ct)
# create the geometry
check_err(lgdal.OGR_F_SetGeometry(feat, ogr_geom._ptr))
else:
# Case where geometry object is not found because of null
# value for field effectively looses whole record in shapefile
# if geometry does not exist
pass
# creat the feature in the layer.
check_err(lgdal.OGR_L_SetFeature(layer, feat))
class LayerMapping:
"A class that maps OGR Layers to GeoDjango Models."
# Acceptable 'base' types for a multi-geometry type.
MULTI_TYPES = {
1: OGRGeomType('MultiPoint'),
2: OGRGeomType('MultiLineString'),
3: OGRGeomType('MultiPolygon'),
OGRGeomType('Point25D').num: OGRGeomType('MultiPoint25D'),
OGRGeomType('LineString25D').num: OGRGeomType('MultiLineString25D'),
OGRGeomType('Polygon25D').num: OGRGeomType('MultiPolygon25D'),
}
# Acceptable Django field types and corresponding acceptable OGR
# counterparts.
FIELD_TYPES = {
models.AutoField: OFTInteger,
models.BigAutoField: OFTInteger64,
models.BooleanField: (OFTInteger, OFTReal, OFTString),
models.IntegerField: (OFTInteger, OFTReal, OFTString),
models.FloatField: (OFTInteger, OFTReal),
models.DateField: OFTDate,
models.DateTimeField: OFTDateTime,
models.EmailField: OFTString,
models.TimeField: OFTTime,
models.DecimalField: (OFTInteger, OFTReal),
models.CharField: OFTString,
models.SlugField: OFTString,
models.TextField: OFTString,
models.URLField: OFTString,
models.UUIDField: OFTString,
models.BigIntegerField: (OFTInteger, OFTReal, OFTString),
models.SmallIntegerField: (OFTInteger, OFTReal, OFTString),
models.PositiveSmallIntegerField: (OFTInteger, OFTReal, OFTString),
}
def __init__(self, model, data, mapping, layer=0,
source_srs=None, encoding='utf-8',
transaction_mode='commit_on_success',
transform=True, unique=None, using=None):
"""
A LayerMapping object is initialized using the given Model (not an instance),
a DataSource (or string path to an OGR-supported data file), and a mapping
dictionary. See the module level docstring for more details and keyword
argument usage.
"""
# Getting the DataSource and the associated Layer.
if isinstance(data, str):
self.ds = DataSource(data, encoding=encoding)
else:
self.ds = data
self.layer = self.ds[layer]
self.using = using if using is not None else router.db_for_write(model)
self.spatial_backend = connections[self.using].ops
# Setting the mapping & model attributes.
self.mapping = mapping
self.model = model
# Checking the layer -- initialization of the object will fail if
# things don't check out before hand.
self.check_layer()
# Getting the geometry column associated with the model (an
# exception will be raised if there is no geometry column).
if connections[self.using].features.supports_transform:
self.geo_field = self.geometry_field()
else:
transform = False
# Checking the source spatial reference system, and getting
# the coordinate transformation object (unless the `transform`
# keyword is set to False)
if transform:
self.source_srs = self.check_srs(source_srs)
self.transform = self.coord_transform()
else:
self.transform = transform
# Setting the encoding for OFTString fields, if specified.
if encoding:
# Making sure the encoding exists, if not a LookupError
# exception will be thrown.
from codecs import lookup
lookup(encoding)
self.encoding = encoding
else:
self.encoding = None
if unique:
self.check_unique(unique)
transaction_mode = 'autocommit' # Has to be set to autocommit.
self.unique = unique
else:
self.unique = None
# Setting the transaction decorator with the function in the
# transaction modes dictionary.
self.transaction_mode = transaction_mode
if transaction_mode == 'autocommit':
self.transaction_decorator = None
elif transaction_mode == 'commit_on_success':
self.transaction_decorator = transaction.atomic
else:
raise LayerMapError('Unrecognized transaction mode: %s' % transaction_mode)
# #### Checking routines used during initialization ####
def check_fid_range(self, fid_range):
"Check the `fid_range` keyword."
if fid_range:
if isinstance(fid_range, (tuple, list)):
return slice(*fid_range)
elif isinstance(fid_range, slice):
return fid_range
else:
raise TypeError
else:
return None
def check_layer(self):
"""
Check the Layer metadata and ensure that it's compatible with the
mapping information and model. Unlike previous revisions, there is no
need to increment through each feature in the Layer.
"""
# The geometry field of the model is set here.
# TODO: Support more than one geometry field / model. However, this
# depends on the GDAL Driver in use.
self.geom_field = False
self.fields = {}
# Getting lists of the field names and the field types available in
# the OGR Layer.
ogr_fields = self.layer.fields
ogr_field_types = self.layer.field_types
# Function for determining if the OGR mapping field is in the Layer.
def check_ogr_fld(ogr_map_fld):
try:
idx = ogr_fields.index(ogr_map_fld)
except ValueError:
raise LayerMapError('Given mapping OGR field "%s" not found in OGR Layer.' % ogr_map_fld)
return idx
# No need to increment through each feature in the model, simply check
# the Layer metadata against what was given in the mapping dictionary.
for field_name, ogr_name in self.mapping.items():
# Ensuring that a corresponding field exists in the model
# for the given field name in the mapping.
try:
model_field = self.model._meta.get_field(field_name)
except FieldDoesNotExist:
raise LayerMapError('Given mapping field "%s" not in given Model fields.' % field_name)
# Getting the string name for the Django field class (e.g., 'PointField').
fld_name = model_field.__class__.__name__
if isinstance(model_field, GeometryField):
if self.geom_field:
raise LayerMapError('LayerMapping does not support more than one GeometryField per model.')
# Getting the coordinate dimension of the geometry field.
coord_dim = model_field.dim
try:
if coord_dim == 3:
gtype = OGRGeomType(ogr_name + '25D')
else:
gtype = OGRGeomType(ogr_name)
except GDALException:
raise LayerMapError('Invalid mapping for GeometryField "%s".' % field_name)
# Making sure that the OGR Layer's Geometry is compatible.
ltype = self.layer.geom_type
if not (ltype.name.startswith(gtype.name) or self.make_multi(ltype, model_field)):
raise LayerMapError('Invalid mapping geometry; model has %s%s, '
'layer geometry type is %s.' %
(fld_name, '(dim=3)' if coord_dim == 3 else '', ltype))
# Setting the `geom_field` attribute w/the name of the model field
# that is a Geometry. Also setting the coordinate dimension
# attribute.
self.geom_field = field_name
self.coord_dim = coord_dim
fields_val = model_field
elif isinstance(model_field, models.ForeignKey):
if isinstance(ogr_name, dict):
# Is every given related model mapping field in the Layer?
rel_model = model_field.remote_field.model
for rel_name, ogr_field in ogr_name.items():
idx = check_ogr_fld(ogr_field)
try:
rel_model._meta.get_field(rel_name)
except FieldDoesNotExist:
raise LayerMapError('ForeignKey mapping field "%s" not in %s fields.' %
(rel_name, rel_model.__class__.__name__))
fields_val = rel_model
else:
raise TypeError('ForeignKey mapping must be of dictionary type.')
else:
# Is the model field type supported by LayerMapping?
if model_field.__class__ not in self.FIELD_TYPES:
raise LayerMapError('Django field type "%s" has no OGR mapping (yet).' % fld_name)
# Is the OGR field in the Layer?
idx = check_ogr_fld(ogr_name)
ogr_field = ogr_field_types[idx]
# Can the OGR field type be mapped to the Django field type?
if not issubclass(ogr_field, self.FIELD_TYPES[model_field.__class__]):
raise LayerMapError('OGR field "%s" (of type %s) cannot be mapped to Django %s.' %
(ogr_field, ogr_field.__name__, fld_name))
fields_val = model_field
self.fields[field_name] = fields_val
def check_srs(self, source_srs):
"Check the compatibility of the given spatial reference object."
if isinstance(source_srs, SpatialReference):
sr = source_srs
elif isinstance(source_srs, self.spatial_backend.spatial_ref_sys()):
sr = source_srs.srs
elif isinstance(source_srs, (int, str)):
sr = SpatialReference(source_srs)
else:
# Otherwise just pulling the SpatialReference from the layer
sr = self.layer.srs
if not sr:
raise LayerMapError('No source reference system defined.')
else:
return sr
def check_unique(self, unique):
"Check the `unique` keyword parameter -- may be a sequence or string."
if isinstance(unique, (list, tuple)):
# List of fields to determine uniqueness with
for attr in unique:
if attr not in self.mapping:
raise ValueError
elif isinstance(unique, str):
# Only a single field passed in.
if unique not in self.mapping:
raise ValueError
else:
raise TypeError('Unique keyword argument must be set with a tuple, list, or string.')
# Keyword argument retrieval routines ####
def feature_kwargs(self, feat):
"""
Given an OGR Feature, return a dictionary of keyword arguments for
constructing the mapped model.
"""
# The keyword arguments for model construction.
kwargs = {}
# Incrementing through each model field and OGR field in the
# dictionary mapping.
for field_name, ogr_name in self.mapping.items():
model_field = self.fields[field_name]
if isinstance(model_field, GeometryField):
# Verify OGR geometry.
try:
val = self.verify_geom(feat.geom, model_field)
except GDALException:
raise LayerMapError('Could not retrieve geometry from feature.')
elif isinstance(model_field, models.base.ModelBase):
# The related _model_, not a field was passed in -- indicating
# another mapping for the related Model.
val = self.verify_fk(feat, model_field, ogr_name)
else:
# Otherwise, verify OGR Field type.
val = self.verify_ogr_field(feat[ogr_name], model_field)
# Setting the keyword arguments for the field name with the
# value obtained above.
kwargs[field_name] = val
return kwargs
def unique_kwargs(self, kwargs):
"""
Given the feature keyword arguments (from `feature_kwargs`), construct
and return the uniqueness keyword arguments -- a subset of the feature
kwargs.
"""
if isinstance(self.unique, str):
return {self.unique: kwargs[self.unique]}
else:
return {fld: kwargs[fld] for fld in self.unique}
# #### Verification routines used in constructing model keyword arguments. ####
def verify_ogr_field(self, ogr_field, model_field):
"""
Verify if the OGR Field contents are acceptable to the model field. If
they are, return the verified value, otherwise raise an exception.
"""
if (isinstance(ogr_field, OFTString) and
isinstance(model_field, (models.CharField, models.TextField))):
if self.encoding and ogr_field.value is not None:
# The encoding for OGR data sources may be specified here
# (e.g., 'cp437' for Census Bureau boundary files).
val = force_str(ogr_field.value, self.encoding)
else:
val = ogr_field.value
if model_field.max_length and val is not None and len(val) > model_field.max_length:
raise InvalidString('%s model field maximum string length is %s, given %s characters.' %
(model_field.name, model_field.max_length, len(val)))
elif isinstance(ogr_field, OFTReal) and isinstance(model_field, models.DecimalField):
try:
# Creating an instance of the Decimal value to use.
d = Decimal(str(ogr_field.value))
except DecimalInvalidOperation:
raise InvalidDecimal('Could not construct decimal from: %s' % ogr_field.value)
# Getting the decimal value as a tuple.
dtup = d.as_tuple()
digits = dtup[1]
d_idx = dtup[2] # index where the decimal is
# Maximum amount of precision, or digits to the left of the decimal.
max_prec = model_field.max_digits - model_field.decimal_places
# Getting the digits to the left of the decimal place for the
# given decimal.
if d_idx < 0:
n_prec = len(digits[:d_idx])
else:
n_prec = len(digits) + d_idx
# If we have more than the maximum digits allowed, then throw an
# InvalidDecimal exception.
if n_prec > max_prec:
raise InvalidDecimal(
'A DecimalField with max_digits %d, decimal_places %d must '
'round to an absolute value less than 10^%d.' %
(model_field.max_digits, model_field.decimal_places, max_prec)
)
val = d
elif isinstance(ogr_field, (OFTReal, OFTString)) and isinstance(model_field, models.IntegerField):
# Attempt to convert any OFTReal and OFTString value to an OFTInteger.
try:
val = int(ogr_field.value)
except ValueError:
raise InvalidInteger('Could not construct integer from: %s' % ogr_field.value)
else:
val = ogr_field.value
return val
def verify_fk(self, feat, rel_model, rel_mapping):
"""
Given an OGR Feature, the related model and its dictionary mapping,
retrieve the related model for the ForeignKey mapping.
"""
# TODO: It is expensive to retrieve a model for every record --
# explore if an efficient mechanism exists for caching related
# ForeignKey models.
# Constructing and verifying the related model keyword arguments.
fk_kwargs = {}
for field_name, ogr_name in rel_mapping.items():
fk_kwargs[field_name] = self.verify_ogr_field(feat[ogr_name], rel_model._meta.get_field(field_name))
# Attempting to retrieve and return the related model.
try:
return rel_model.objects.using(self.using).get(**fk_kwargs)
except ObjectDoesNotExist:
raise MissingForeignKey(
'No ForeignKey %s model found with keyword arguments: %s' %
(rel_model.__name__, fk_kwargs)
)
def verify_geom(self, geom, model_field):
"""
Verify the geometry -- construct and return a GeometryCollection
if necessary (for example if the model field is MultiPolygonField while
the mapped shapefile only contains Polygons).
"""
# Downgrade a 3D geom to a 2D one, if necessary.
if self.coord_dim != geom.coord_dim:
geom.coord_dim = self.coord_dim
if self.make_multi(geom.geom_type, model_field):
# Constructing a multi-geometry type to contain the single geometry
multi_type = self.MULTI_TYPES[geom.geom_type.num]
g = OGRGeometry(multi_type)
g.add(geom)
else:
g = geom
# Transforming the geometry with our Coordinate Transformation object,
# but only if the class variable `transform` is set w/a CoordTransform
# object.
if self.transform:
g.transform(self.transform)
# Returning the WKT of the geometry.
return g.wkt
# #### Other model methods ####
def coord_transform(self):
"Return the coordinate transformation object."
SpatialRefSys = self.spatial_backend.spatial_ref_sys()
try:
# Getting the target spatial reference system
target_srs = SpatialRefSys.objects.using(self.using).get(srid=self.geo_field.srid).srs
# Creating the CoordTransform object
return CoordTransform(self.source_srs, target_srs)
except Exception as exc:
raise LayerMapError(
'Could not translate between the data source and model geometry.'
) from exc
def geometry_field(self):
"Return the GeometryField instance associated with the geographic column."
# Use `get_field()` on the model's options so that we
# get the correct field instance if there's model inheritance.
opts = self.model._meta
return opts.get_field(self.geom_field)
def make_multi(self, geom_type, model_field):
"""
Given the OGRGeomType for a geometry and its associated GeometryField,
determine whether the geometry should be turned into a GeometryCollection.
"""
return (geom_type.num in self.MULTI_TYPES and
model_field.__class__.__name__ == 'Multi%s' % geom_type.django)
def save(self, verbose=False, fid_range=False, step=False,
progress=False, silent=False, stream=sys.stdout, strict=False):
"""
Save the contents from the OGR DataSource Layer into the database
according to the mapping dictionary given at initialization.
Keyword Parameters:
verbose:
If set, information will be printed subsequent to each model save
executed on the database.
fid_range:
May be set with a slice or tuple of (begin, end) feature ID's to map
from the data source. In other words, this keyword enables the user
to selectively import a subset range of features in the geographic
data source.
step:
If set with an integer, transactions will occur at every step
interval. For example, if step=1000, a commit would occur after
the 1,000th feature, the 2,000th feature etc.
progress:
When this keyword is set, status information will be printed giving
the number of features processed and successfully saved. By default,
progress information will pe printed every 1000 features processed,
however, this default may be overridden by setting this keyword with an
integer for the desired interval.
stream:
Status information will be written to this file handle. Defaults to
using `sys.stdout`, but any object with a `write` method is supported.
silent:
By default, non-fatal error notifications are printed to stdout, but
this keyword may be set to disable these notifications.
strict:
Execution of the model mapping will cease upon the first error
encountered. The default behavior is to attempt to continue.
"""
# Getting the default Feature ID range.
default_range = self.check_fid_range(fid_range)
# Setting the progress interval, if requested.
if progress:
if progress is True or not isinstance(progress, int):
progress_interval = 1000
else:
progress_interval = progress
def _save(feat_range=default_range, num_feat=0, num_saved=0):
if feat_range:
layer_iter = self.layer[feat_range]
else:
layer_iter = self.layer
for feat in layer_iter:
num_feat += 1
# Getting the keyword arguments
try:
kwargs = self.feature_kwargs(feat)
except LayerMapError as msg:
# Something borked the validation
if strict:
raise
elif not silent:
stream.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg))
else:
# Constructing the model using the keyword args
is_update = False
if self.unique:
# If we want unique models on a particular field, handle the
# geometry appropriately.
try:
# Getting the keyword arguments and retrieving
# the unique model.
u_kwargs = self.unique_kwargs(kwargs)
m = self.model.objects.using(self.using).get(**u_kwargs)
is_update = True
# Getting the geometry (in OGR form), creating
# one from the kwargs WKT, adding in additional
# geometries, and update the attribute with the
# just-updated geometry WKT.
geom_value = getattr(m, self.geom_field)
if geom_value is None:
geom = OGRGeometry(kwargs[self.geom_field])
else:
geom = geom_value.ogr
new = OGRGeometry(kwargs[self.geom_field])
for g in new:
geom.add(g)
setattr(m, self.geom_field, geom.wkt)
except ObjectDoesNotExist:
# No unique model exists yet, create.
m = self.model(**kwargs)
else:
m = self.model(**kwargs)
try:
# Attempting to save.
m.save(using=self.using)
num_saved += 1
if verbose:
stream.write('%s: %s\n' % ('Updated' if is_update else 'Saved', m))
except Exception as msg:
if strict:
# Bailing out if the `strict` keyword is set.
if not silent:
stream.write(
'Failed to save the feature (id: %s) into the '
'model with the keyword arguments:\n' % feat.fid
)
stream.write('%s\n' % kwargs)
raise
elif not silent:
stream.write('Failed to save %s:\n %s\nContinuing\n' % (kwargs, msg))
# Printing progress information, if requested.
if progress and num_feat % progress_interval == 0:
stream.write('Processed %d features, saved %d …\n' % (num_feat, num_saved))
# Only used for status output purposes -- incremental saving uses the
# values returned here.
return num_saved, num_feat
if self.transaction_decorator is not None:
_save = self.transaction_decorator(_save)
nfeat = self.layer.num_feat
if step and isinstance(step, int) and step < nfeat:
# Incremental saving is requested at the given interval (step)
if default_range:
raise LayerMapError('The `step` keyword may not be used in conjunction with the `fid_range` keyword.')
beg, num_feat, num_saved = (0, 0, 0)
indices = range(step, nfeat, step)
n_i = len(indices)
for i, end in enumerate(indices):
# Constructing the slice to use for this step; the last slice is
# special (e.g, [100:] instead of [90:100]).
if i + 1 == n_i:
step_slice = slice(beg, None)
else:
step_slice = slice(beg, end)
try:
num_feat, num_saved = _save(step_slice, num_feat, num_saved)
beg = end
except Exception: # Deliberately catch everything
stream.write('%s\nFailed to save slice: %s\n' % ('=-' * 20, step_slice))
raise
else:
# Otherwise, just calling the previously defined _save() function.
_save()
def import_geodata(data_files, data_path, catalog):
"""
Imports the geospatial data into the database.
Note, in the case of this data, there's data attached to the multipolygon that we want in the datastore.
Will also convert Polygons to MulitPolygons.
Not idempotent! (get_or_create doesn't really work with geometry objects.)
Args:
data_files (list(str)): List of source files.
data_path (path): Path to the source files.
catalog (Catalog): Catalog that this geodata belongs to.
"""
geometry_data_mapping = {
'CUSEC': 'cusec',
'CUMUN': 'cumun',
'CSEC': 'secc',
'CDIS': 'dist',
'CMUN': 'cmun',
'CPRO': 'cpro',
'CCA': 'ccaa',
'CUDIS': 'cudis',
'OBS': 'obs',
'CNUT0': 'cnut0',
'CNUT1': 'cnut1',
'CNUT2': 'cnut2',
'CNUT3': 'cnut3',
'CLAU2': 'clau2',
'NPRO': 'npro',
'NCA': 'nca',
'NMUN': 'nmun'
}
metadata_map = {
"OBJECTID": "obj_id",
"Shape_len": "perimeter",
"Shape_area": "area",
}
ignore = ["Shape_Leng"]
field_skip = list(metadata_map.keys()) + ignore
metadata_skip = list(geometry_data_mapping.keys()) + ignore
shp = [s for s in data_files if ".shp" == s[-4:]][0]
shape_file = path.join(data_path, shp)
ds = DataSource(shape_file)
layer = ds[0]
fields = layer.fields
source_srid = SpatialReference(layer.srs.srid)
our_srid = SpatialReference(4326)
transform = CoordTransform(source_srid, our_srid)
for shape in layer:
geom = shape.geom
geom.transform(transform)
field_data = {
geometry_data_mapping[k]: shape.get(v)
for k, v in zip(fields, range(len(fields))) if k not in field_skip
}
metadata = {
metadata_map[k]: shape.get(v)
for k, v in zip(fields, range(len(fields)))
if k not in metadata_skip
}
# Normalise all Polygons to Multipolygons for ease of use later.
if geom.geom_type.name == "Polygon":
mp = OGRGeometry(OGRGeomType('MultiPolygon'))
mp.add(geom)
geom = mp
g = GeometryStore(catalog=catalog, geom=geom.geos, metadata=metadata)
g.save()
d = DataStore(catalog=catalog, parent_geometry=g, data=field_data)
d.save()
