def compute_max_zoom(self):
"""
Set max zoom property based on rasterlayer metadata.
"""
# Return manual override value if provided
if self.rasterlayer.max_zoom is not None:
return self.rasterlayer.max_zoom
if self.dataset.srs.srid == WEB_MERCATOR_SRID:
# For rasters in web mercator, use the scale directly
scale = abs(self.dataset.scale.x)
else:
# Create a line from the center of the raster to a point that is
# one pixel width away from the center.
xcenter = self.dataset.extent[0] + (self.dataset.extent[2] - self.dataset.extent[0]) / 2
ycenter = self.dataset.extent[1] + (self.dataset.extent[3] - self.dataset.extent[1]) / 2
linestring = 'LINESTRING({} {}, {} {})'.format(
xcenter, ycenter, xcenter + self.dataset.scale.x, ycenter
)
line = OGRGeometry(linestring, srs=self.dataset.srs)
# Tansform the line into web mercator.
line.transform(WEB_MERCATOR_SRID)
# Use the lenght of the transformed line as scale.
scale = line.geos.length
return utils.closest_zoomlevel(scale)
def set_simple_polylines(self, target_field_name='simple_mpoly', tolerance=2, srid=4326):
"""
Ben Welsh method
Simplifies the source polygons so they don't use so many points.
Provide a tolerance score the indicates how sharply the
the lines should be redrawn.
Returns True if successful.
"""
# Fetch the source polygon
source_field_name = 'geom'
source = getattr(self, source_field_name)
# Fetch the target polygon where the result will be saved
#target_field_name = 'simple_mpoly'
target = getattr(self, target_field_name)
# Simplify the source
simple = source.transform(srid, True).simplify(tolerance, True)
# If it's a polyline, convert it to a MultiPolyline
if simple.geom_type == 'LineString':
mp = OGRGeometry(OGRGeomType('MultiLineString'))
mp.add(simple.wkt)
target = mp.wkt
# Otherwise just save out right away
else:
target = simple.wkt
# Set the attribute
setattr(self, target_field_name, target)
# Save out
self.save()
return True
def get_wkt(value, srid=DEFAULT_PROJ):
"""
`value` is either a WKT string or a geometry field. Returns WKT in the
projection for the given SRID.
"""
ogr = None
if value:
if isinstance(value, OGRGeometry):
ogr = value
elif isinstance(value, GEOSGeometry):
ogr = value.ogr
elif isinstance(value, basestring):
match = ewkt_re.match(value)
if match:
ogr = OGRGeometry(match.group('wkt'), match.group('srid'))
else:
ogr = OGRGeometry(value)
wkt = ''
if ogr:
# Workaround for Django bug #12312. GEOSGeometry types don't support 3D wkt;
# OGRGeometry types output 3D for linestrings even if they should do 2D, causing
# IntegrityError's.
if ogr.dimension == 2:
geos = ogr.geos
geos.transform(srid)
wkt = geos.wkt
else:
ogr.transform(srid)
wkt = ogr.wkt
return wkt
def region_shapefile(filename, year):
'''Import the regions'''
census = models.Census.objects.get_or_create(year=year)[0]
ds = DataSource(filename)
lyr = ds[0]
spatial_backend = connections[router.db_for_write(models.Meshblock)].ops
SpatialRefSys = spatial_backend.spatial_ref_sys()
target_srs = SpatialRefSys.objects.get(srid=models.Region._meta.get_field_by_name('area')[0].srid).srs
transform = CoordTransform(lyr.srs, target_srs)
for mb_item in lyr:
with transaction.atomic():
region = models.Region(census=census)
g = OGRGeometry(OGRGeomType('MultiPolygon'))
g.add(mb_item.geom)
g.transform(transform)
mpg = MultiPolygon([Polygon(*[[(x, y) for (x, y, z) in inner] for inner in middle], srid=lyr.srs.srid) for middle in g.coords], srid=lyr.srs.srid)
region.area = mpg.ewkt
region.region_name = mb_item['REGC2013_N']
region.save()
def extent(self, srid=WEB_MERCATOR_SRID):
"""
Returns bbox for layer.
"""
if not self._bbox or self._bbox_srid != srid:
# Get bbox for raster in original coordinates
meta = self.metadata
xmin = meta.uperleftx
ymax = meta.uperlefty
xmax = xmin + meta.width * meta.scalex
ymin = ymax + meta.height * meta.scaley
# Create Polygon box
geom = OGRGeometry(Envelope((xmin, ymin, xmax, ymax)).wkt)
# Set original srs
if meta.srs_wkt:
geom.srs = SpatialReference(meta.srs_wkt)
else:
geom.srid = meta.srid
# Transform to requested srid
geom.transform(srid)
# Calculate value range for bbox
coords = geom.coords[0]
xvals = [x[0] for x in coords]
yvals = [x[1] for x in coords]
# Set bbox
self._bbox = (min(xvals), min(yvals), max(xvals), max(yvals))
self._bbox_srid = srid
return self._bbox
def verify_geom(self, geom, model_field):
"""
Verifies the geometry -- will construct and return a GeometryCollection
if necessary (for example if the model field is MultiPolygonField while
the mapped shapefile only contains Polygons).
"""
# Downgrade a 3D geom to a 2D one, if necessary.
if self.coord_dim != geom.coord_dim:
geom.coord_dim = self.coord_dim
if self.make_multi(geom.geom_type, model_field):
# Constructing a multi-geometry type to contain the single geometry
multi_type = self.MULTI_TYPES[geom.geom_type.num]
g = OGRGeometry(multi_type)
g.add(geom)
else:
g = geom
# Transforming the geometry with our Coordinate Transformation object,
# but only if the class variable `transform` is set w/a CoordTransform
# object.
if self.transform: g.transform(self.transform)
# Returning the WKT of the geometry.
return g.wkt
def _get_multipolygon_geometry_from_row(self, row):
if row.geom_type.django == 'PolygonField':
geom = OGRGeometry(OGRGeomType('MultiPolygon'))
geom.add(row.geom)
return geom
elif row.geom_type.django == 'MultiPolygonField':
return geom
def handle(self, *args, **kwargs):
shape_file = args[0]
ds = DataSource(shape_file)
layer = ds[0]
for feat in layer:
try:
area_id = feat['ID_0'].value
except:
area_id = feat['GADMID'].value
area_name = unicode(feat['NAME_ENGLI'].value, 'iso-8859-1')
area_varname = unicode(feat['NAME_LOCAL'].value, 'iso-8859-1')
mpgeom = OGRGeometry('MultiPolygon')
mpgeom.add(feat.geom)
area_geom = GEOSGeometry(mpgeom.wkt)
print "%s %s %s" % (area_id, area_name, area_varname)
area = Area()
area.shape_id = area_id
area.name = area_name
area.varname = area_varname
area.type = 'Country'
area.save()
area.update_path()
area.save()
print "Tree ID: %d" % (area.tree_id,)
areageom = Geom(area=area)
areageom.geom = area_geom
areageom.save()
def area_shapefile(filename, year):
"""Import the area units"""
census = models.Census.objects.get_or_create(year=year)[0]
ds = DataSource(filename)
lyr = ds[0]
spatial_backend = connections[router.db_for_write(models.Meshblock)].ops
SpatialRefSys = spatial_backend.spatial_ref_sys()
target_srs = SpatialRefSys.objects.get(srid=models.Area._meta.get_field_by_name("area")[0].srid).srs
transform = CoordTransform(lyr.srs, target_srs)
for mb_item in lyr:
with transaction.atomic():
area = models.Area(census=census)
g = OGRGeometry(OGRGeomType("MultiPolygon"))
g.add(mb_item.geom)
g.transform(transform)
mpg = MultiPolygon(
[
Polygon(*[[(x, y) for (x, y, z) in inner] for inner in middle], srid=lyr.srs.srid)
for middle in g.coords
],
srid=lyr.srs.srid,
)
area.area = mpg.ewkt
area.area_name = mb_item["AU2013_NAM"]
area.save()
def transform_rd(point):
''' Please note this returns an OGRGeometry, not a point '''
src_string = '+proj=sterea +lat_0=52.15616055555555 +lon_0=5.38763888888889 +k=0.9999079 +x_0=155000 +y_0=463000 +ellps=bessel +units=m +towgs84=565.2369,50.0087,465.658,-0.406857330322398,0.350732676542563,-1.8703473836068,4.0812 +no_defs no_defs <>'
src_srs = SpatialReference(src_string)
geom = OGRGeometry(point.wkt, src_srs)
geom.transform(4326)
return geom
def test07b_closepolygons(self):
"Testing closing Polygon objects."
# Both rings in this geometry are not closed.
poly = OGRGeometry("POLYGON((0 0, 5 0, 5 5, 0 5), (1 1, 2 1, 2 2, 2 1))")
self.assertEqual(8, poly.point_count)
print "\nBEGIN - expecting IllegalArgumentException; safe to ignore.\n"
try:
c = poly.centroid
except OGRException:
# Should raise an OGR exception, rings are not closed
pass
else:
self.fail("Should have raised an OGRException!")
print "\nEND - expecting IllegalArgumentException; safe to ignore.\n"
# Closing the rings -- doesn't work on GDAL versions 1.4.1 and below:
# http://trac.osgeo.org/gdal/ticket/1673
major, minor1, minor2 = gdal_version().split(".")
if major == "1":
iminor1 = int(minor1)
if iminor1 < 4 or (iminor1 == 4 and minor2.startswith("1")):
return
poly.close_rings()
self.assertEqual(10, poly.point_count) # Two closing points should've been added
self.assertEqual(OGRGeometry("POINT(2.5 2.5)"), poly.centroid)
def transform(self, ct, clone=False):
"""
Requires GDAL. Transforms the geometry according to the given
transformation object, which may be an integer SRID, and WKT or
PROJ.4 string. By default, the geometry is transformed in-place and
nothing is returned. However if the `clone` keyword is set, then this
geometry will not be modified and a transformed clone will be returned
instead.
"""
srid = self.srid
if HAS_GDAL and srid:
g = OGRGeometry(self.wkb, srid)
g.transform(ct)
wkb = str(g.wkb)
ptr = from_wkb(wkb, len(wkb))
if clone:
# User wants a cloned transformed geometry returned.
return GEOSGeometry(ptr, srid=g.srid)
if ptr:
# Reassigning pointer, and performing post-initialization setup
# again due to the reassignment.
destroy_geom(self.ptr)
self._ptr = ptr
self._post_init(g.srid)
else:
raise GEOSException('Transformed WKB was invalid.')
def convOSM(wkt):
""" Converts standard merkartor
to osm projection as tuple
"""
obj = OGRGeometry(wkt)
obj.srs = 'EPSG:4326'
obj.transform_to(SpatialReference('EPSG:900913'))
#obj.transform_to(SpatialReference('EPSG:4326'))
return (obj.x, obj.y)
def set_simple_linestrings(self, tolerance=500):
"""
Simplifies the source linestrings so they don't use so many points.
Provide a tolerance score the indicates how sharply the
the lines should be redrawn.
Returns True if successful.
"""
# Get the list of SRIDs we need to update
srid_list = self.get_srid_list()
# Loop through each
for srid in srid_list:
# Fetch the source polygon
source_field_name = 'linestring_%s' % str(srid)
source = getattr(self, source_field_name)
# Fetch the target polygon where the result will be saved
target_field_name = 'simple_%s' % source_field_name
# If there's nothing to transform, drop out now.
if not source:
setattr(self, target_field_name, None)
continue
if srid != 900913:
# Transform the source out of lng/lat before the simplification
copy = source.transform(900913, clone=True)
else:
copy = deepcopy(source)
# Simplify the source
simple = copy.simplify(tolerance, True)
# If the result is a polygon ...
if simple.geom_type == 'LineString':
# Create a new Multipolygon shell
ml = OGRGeometry(OGRGeomType('MultiLineString'))
# Transform the new poly back to its SRID
simple.transform(srid)
# Stuff it in the shell
ml.add(simple.wkt)
# Grab the WKT
target = ml.wkt
# If it's not a polygon...
else:
# It should be ready to go, so transform
simple.transform(srid)
# And grab the WKT
target = simple.wkt
# Stuff the WKT into the field
setattr(self, target_field_name, target)
return True
def merge_geometries(geometries_str, sep='$'):
"""Take a list of geometries in a string, and merge it."""
geometries = geometries_str.split(sep)
if len(geometries) == 1:
return geometries_str
else:
pool = OGRGeometry(geometries[0])
for geom in geometries:
pool = pool.union(OGRGeometry(geom))
return pool.wkt
def test_srs(self):
"Testing OGR Geometries with Spatial Reference objects."
for mp in self.geometries.multipolygons:
# Creating a geometry w/spatial reference
sr = SpatialReference("WGS84")
mpoly = OGRGeometry(mp.wkt, sr)
self.assertEqual(sr.wkt, mpoly.srs.wkt)
# Ensuring that SRS is propagated to clones.
klone = mpoly.clone()
self.assertEqual(sr.wkt, klone.srs.wkt)
# Ensuring all children geometries (polygons and their rings) all
# return the assigned spatial reference as well.
for poly in mpoly:
self.assertEqual(sr.wkt, poly.srs.wkt)
for ring in poly:
self.assertEqual(sr.wkt, ring.srs.wkt)
# Ensuring SRS propagate in topological ops.
a = OGRGeometry(self.geometries.topology_geoms[0].wkt_a, sr)
b = OGRGeometry(self.geometries.topology_geoms[0].wkt_b, sr)
diff = a.difference(b)
union = a.union(b)
self.assertEqual(sr.wkt, diff.srs.wkt)
self.assertEqual(sr.srid, union.srs.srid)
# Instantiating w/an integer SRID
mpoly = OGRGeometry(mp.wkt, 4326)
self.assertEqual(4326, mpoly.srid)
mpoly.srs = SpatialReference(4269)
self.assertEqual(4269, mpoly.srid)
self.assertEqual("NAD83", mpoly.srs.name)
# Incrementing through the multipolygon after the spatial reference
# has been re-assigned.
for poly in mpoly:
self.assertEqual(mpoly.srs.wkt, poly.srs.wkt)
poly.srs = 32140
for ring in poly:
# Changing each ring in the polygon
self.assertEqual(32140, ring.srs.srid)
self.assertEqual("NAD83 / Texas South Central", ring.srs.name)
ring.srs = str(SpatialReference(4326)) # back to WGS84
self.assertEqual(4326, ring.srs.srid)
# Using the `srid` property.
ring.srid = 4322
self.assertEqual("WGS 72", ring.srs.name)
self.assertEqual(4322, ring.srid)
# srs/srid may be assigned their own values, even when srs is None.
mpoly = OGRGeometry(mp.wkt, srs=None)
mpoly.srs = mpoly.srs
mpoly.srid = mpoly.srid
def street_location(request):
if request.method == 'GET':
params = request.GET
points = []
if params.has_key('str'):
if params.has_key('int'):
points += (get_locations_by_intersection(params['str'],params['int']))
elif params.has_key('door'):
loc = get_location_by_door(params['str'],params['door'])
if loc:
points.append(loc[0].wkt)
else:
raise Http404
if params.has_key('out'):
out = params['out']
if out == 'layer' and points:
lpoints = []
for p in points:
pgeom = OGRGeometry(p)
pgeom.srs = 'EPSG:4326'
pgeom.transform_to(SpatialReference('EPSG:900913'))
pcoord = simplejson.loads(pgeom.json)
lpoints.append({
"type": "point",
"id": 'point',
"name": "pipin",
"geojson": pcoord,
"icon": {
"url": "/media/icons/info.png",
"width": 32,
"height": 37
}
})
layer = {
'type': 'layer',
'id': 'layer-location',
'elements': lpoints,
'box_size': None
}
return HttpResponse(simplejson.dumps(layer), mimetype='text/json')
else:
raise Http404
else:
path = request.get_full_path() + '&out=layer'
context = RequestContext(request, {'params': params, 'layerpath':path})
return render_to_response('maap/street_location.html', context_instance=context)
else:
raise Http404
else:
raise Http404
def test_symdifference(self):
"Testing sym_difference()."
for i in range(len(self.geometries.topology_geoms)):
a = OGRGeometry(self.geometries.topology_geoms[i].wkt_a)
b = OGRGeometry(self.geometries.topology_geoms[i].wkt_b)
d1 = OGRGeometry(self.geometries.sdiff_geoms[i].wkt)
d2 = a.sym_difference(b)
self.assertEqual(d1, d2)
self.assertEqual(d1, a ^ b) # __xor__ is symmetric difference operator
a ^= b # testing __ixor__
self.assertEqual(d1, a)
def test_closepolygons(self):
"Testing closing Polygon objects."
# Both rings in this geometry are not closed.
poly = OGRGeometry("POLYGON((0 0, 5 0, 5 5, 0 5), (1 1, 2 1, 2 2, 2 1))")
self.assertEqual(8, poly.point_count)
with self.assertRaises(GDALException):
poly.centroid
poly.close_rings()
self.assertEqual(10, poly.point_count) # Two closing points should've been added
self.assertEqual(OGRGeometry("POINT(2.5 2.5)"), poly.centroid)
def test_transform_dim(self):
"Testing coordinate dimension is the same on transformed geometries."
ls_orig = OGRGeometry("LINESTRING(-104.609 38.255)", 4326)
ls_trans = OGRGeometry("LINESTRING(992385.4472045 481455.4944650)", 2774)
prec = 3
ls_orig.transform(ls_trans.srs)
# Making sure the coordinate dimension is still 2D.
self.assertEqual(2, ls_orig.coord_dim)
self.assertAlmostEqual(ls_trans.x[0], ls_orig.x[0], prec)
self.assertAlmostEqual(ls_trans.y[0], ls_orig.y[0], prec)
def test_difference(self):
"Testing difference()."
for i in range(len(self.geometries.topology_geoms)):
a = OGRGeometry(self.geometries.topology_geoms[i].wkt_a)
b = OGRGeometry(self.geometries.topology_geoms[i].wkt_b)
d1 = OGRGeometry(self.geometries.diff_geoms[i].wkt)
d2 = a.difference(b)
self.assertEqual(d1, d2)
self.assertEqual(d1, a - b) # __sub__ is difference operator
a -= b # testing __isub__
self.assertEqual(d1, a)
def test_union(self):
"Testing union()."
for i in range(len(self.geometries.topology_geoms)):
a = OGRGeometry(self.geometries.topology_geoms[i].wkt_a)
b = OGRGeometry(self.geometries.topology_geoms[i].wkt_b)
u1 = OGRGeometry(self.geometries.union_geoms[i].wkt)
u2 = a.union(b)
self.assertEqual(u1, u2)
self.assertEqual(u1, a | b) # __or__ is union operator
a |= b # testing __ior__
self.assertEqual(u1, a)
def test_intersection(self):
"Testing intersects() and intersection()."
for i in range(len(self.geometries.topology_geoms)):
a = OGRGeometry(self.geometries.topology_geoms[i].wkt_a)
b = OGRGeometry(self.geometries.topology_geoms[i].wkt_b)
i1 = OGRGeometry(self.geometries.intersect_geoms[i].wkt)
self.assertTrue(a.intersects(b))
i2 = a.intersection(b)
self.assertEqual(i1, i2)
self.assertEqual(i1, a & b) # __and__ is intersection operator
a &= b # testing __iand__
self.assertEqual(i1, a)
def geometry_to_multipolygon(geometry):
"""
Converts a Polygon to a MultiPolygon.
"""
if geometry.__class__.__name__ == 'MultiPolygon':
return geometry
elif geometry.__class__.__name__ == 'Polygon':
multipolygon = OGRGeometry(OGRGeomType('MultiPolygon'))
multipolygon.add(geometry)
return multipolygon
else:
raise ValueError(_('The geometry is neither a Polygon nor a MultiPolygon.'))
def Get_PointGeom(d):
if d['g']:
X = d['X']
Y = d['Y']
Z = d['Z']
wkt = 'POINT(%f %f)'%(X,Y) # z niet ondersteund in geometry kennelijk...
oPoint = OGRGeometry(wkt, SpatialReference('EPSG:28992'))
oPoint.transform_to(SpatialReference('WGS84')) # transform van RD_NEW naar WGS84
PointGeom = oPoint.wkt
return PointGeom
else:
return None
def test10_difference(self):
"Testing difference()."
for i in xrange(len(topology_geoms)):
g_tup = topology_geoms[i]
a = OGRGeometry(g_tup[0].wkt)
b = OGRGeometry(g_tup[1].wkt)
d1 = OGRGeometry(diff_geoms[i].wkt)
d2 = a.difference(b)
self.assertEqual(d1, d2)
self.assertEqual(d1, a - b) # __sub__ is difference operator
a -= b # testing __isub__
self.assertEqual(d1, a)
def test13_union(self):
"Testing union()."
for i in xrange(len(topology_geoms)):
g_tup = topology_geoms[i]
a = OGRGeometry(g_tup[0].wkt)
b = OGRGeometry(g_tup[1].wkt)
u1 = OGRGeometry(union_geoms[i].wkt)
u2 = a.union(b)
self.assertEqual(u1, u2)
self.assertEqual(u1, a | b) # __or__ is union operator
a |= b # testing __ior__
self.assertEqual(u1, a)
def test12_symdifference(self):
"Testing sym_difference()."
for i in xrange(len(topology_geoms)):
g_tup = topology_geoms[i]
a = OGRGeometry(g_tup[0].wkt)
b = OGRGeometry(g_tup[1].wkt)
d1 = OGRGeometry(sdiff_geoms[i].wkt)
d2 = a.sym_difference(b)
self.assertEqual(d1, d2)
self.assertEqual(d1, a ^ b) # __xor__ is symmetric difference operator
a ^= b # testing __ixor__
self.assertEqual(d1, a)
def polygon_to_multipolygon(self, geom):
"""
Convert polygons to multipolygons so all features are homogenous in the database.
"""
if geom.__class__.__name__ == 'Polygon':
g = OGRGeometry(OGRGeomType('MultiPolygon'))
g.add(geom)
return g
elif geom.__class__.__name__ == 'MultiPolygon':
return geom
else:
raise ValueError('Geom is neither Polygon nor MultiPolygon.')
def merge(self, other):
"""
Creates a new MultiPolygon from the Polygons of two MultiPolygons.
"""
if hasattr(other, 'geometry'):
other = other.geometry
geometry = OGRGeometry(OGRGeomType('MultiPolygon'))
for polygon in self.geometry:
geometry.add(polygon)
for polygon in other:
geometry.add(polygon)
return Geometry(geometry)
def test_touches(self):
self.assertIs(
OGRGeometry('POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))').touches(OGRGeometry('LINESTRING(0 2, 2 0)')), True
)
self.assertIs(OGRGeometry('POINT(0 0)').touches(OGRGeometry('POINT(0 1)')), False)
def process(self):
"""
This method contains the logic for processing tasks asynchronously
from a background thread or from the scheduler. Here tasks that are
ready to be processed execute some logic. This could be communication
with a processing node or executing a pending action.
"""
try:
if self.auto_processing_node and not self.status in [
status_codes.FAILED, status_codes.CANCELED
]:
# No processing node assigned and need to auto assign
if self.processing_node is None:
# Assign first online node with lowest queue count
self.processing_node = ProcessingNode.find_best_available_node(
)
if self.processing_node:
self.processing_node.queue_count += 1 # Doesn't have to be accurate, it will get overriden later
self.processing_node.save()
logger.info(
"Automatically assigned processing node {} to {}".
format(self.processing_node, self))
self.save()
# Processing node assigned, but is offline and no errors
if self.processing_node and not self.processing_node.is_online(
):
# Detach processing node, will be processed at the next tick
logger.info(
"Processing node {} went offline, reassigning {}...".
format(self.processing_node, self))
self.uuid = ''
self.processing_node = None
self.save()
if self.processing_node:
# Need to process some images (UUID not yet set and task doesn't have pending actions)?
if not self.uuid and self.pending_action is None and self.status is None:
logger.info("Processing... {}".format(self))
images = [
image.path() for image in self.imageupload_set.all()
]
# This takes a while
uuid = self.processing_node.process_new_task(
images, self.name, self.options)
# Refresh task object before committing change
self.refresh_from_db()
self.uuid = uuid
self.save()
# TODO: log process has started processing
if self.pending_action is not None:
if self.pending_action == pending_actions.CANCEL:
# Do we need to cancel the task on the processing node?
logger.info("Canceling {}".format(self))
if self.processing_node and self.uuid:
# Attempt to cancel the task on the processing node
# We don't care if this fails (we tried)
try:
self.processing_node.cancel_task(self.uuid)
self.status = None
except ProcessingException:
logger.warning(
"Could not cancel {} on processing node. We'll proceed anyway..."
.format(self))
self.status = status_codes.CANCELED
self.pending_action = None
self.save()
else:
raise ProcessingError(
"Cannot cancel a task that has no processing node or UUID"
)
elif self.pending_action == pending_actions.RESTART:
logger.info("Restarting {}".format(self))
if self.processing_node:
# Check if the UUID is still valid, as processing nodes purge
# results after a set amount of time, the UUID might have eliminated.
uuid_still_exists = False
if self.uuid:
try:
info = self.processing_node.get_task_info(
self.uuid)
uuid_still_exists = info['uuid'] == self.uuid
except ProcessingException:
pass
if uuid_still_exists:
# Good to go
try:
self.processing_node.restart_task(self.uuid)
except ProcessingError as e:
# Something went wrong
logger.warning(
"Could not restart {}, will start a new one"
.format(self))
self.uuid = ''
else:
# Task has been purged (or processing node is offline)
# Process this as a new task
# Removing its UUID will cause the scheduler
# to process this the next tick
self.uuid = ''
self.console_output = ""
self.processing_time = -1
self.status = None
self.last_error = None
self.pending_action = None
self.save()
else:
raise ProcessingError(
"Cannot restart a task that has no processing node"
)
elif self.pending_action == pending_actions.REMOVE:
logger.info("Removing {}".format(self))
if self.processing_node and self.uuid:
# Attempt to delete the resources on the processing node
# We don't care if this fails, as resources on processing nodes
# Are expected to be purged on their own after a set amount of time anyway
try:
self.processing_node.remove_task(self.uuid)
except ProcessingException:
pass
# What's more important is that we delete our task properly here
self.delete()
# Stop right here!
return
if self.processing_node:
# Need to update status (first time, queued or running?)
if self.uuid and self.status in [
None, status_codes.QUEUED, status_codes.RUNNING
]:
# Update task info from processing node
info = self.processing_node.get_task_info(self.uuid)
self.processing_time = info["processingTime"]
self.status = info["status"]["code"]
current_lines_count = len(
self.console_output.split("\n")) - 1
self.console_output += self.processing_node.get_task_console_output(
self.uuid, current_lines_count)
if "errorMessage" in info["status"]:
self.last_error = info["status"]["errorMessage"]
# Has the task just been canceled, failed, or completed?
if self.status in [
status_codes.FAILED, status_codes.COMPLETED,
status_codes.CANCELED
]:
logger.info("Processing status: {} for {}".format(
self.status, self))
if self.status == status_codes.COMPLETED:
assets_dir = self.assets_path("")
# Remove previous assets directory
if os.path.exists(assets_dir):
logger.info(
"Removing old assets directory: {} for {}".
format(assets_dir, self))
shutil.rmtree(assets_dir)
os.makedirs(assets_dir)
logger.info(
"Downloading all.zip for {}".format(self))
# Download all assets
zip_stream = self.processing_node.download_task_asset(
self.uuid, "all.zip")
zip_path = os.path.join(assets_dir, "all.zip")
with open(zip_path, 'wb') as fd:
for chunk in zip_stream.iter_content(4096):
fd.write(chunk)
logger.info(
"Done downloading all.zip for {}".format(self))
# Extract from zip
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
logger.info(
"Extracted all.zip for {}".format(self))
# Populate *_extent fields
extent_fields = [
(os.path.realpath(
self.assets_path("odm_orthophoto",
"odm_orthophoto.tif")),
'orthophoto_extent'),
(os.path.realpath(
self.assets_path("odm_dem", "dsm.tif")),
'dsm_extent'),
(os.path.realpath(
self.assets_path("odm_dem", "dtm.tif")),
'dtm_extent'),
]
for raster_path, field in extent_fields:
if os.path.exists(raster_path):
# Read extent and SRID
raster = GDALRaster(raster_path)
extent = OGRGeometry.from_bbox(
raster.extent)
# It will be implicitly transformed into the SRID of the model’s field
# self.field = GEOSGeometry(...)
setattr(
self, field,
GEOSGeometry(extent.wkt,
srid=raster.srid))
logger.info(
"Populated extent field with {} for {}"
.format(raster_path, self))
self.update_available_assets_field()
self.save()
else:
# FAILED, CANCELED
self.save()
else:
# Still waiting...
self.save()
except ProcessingError as e:
self.set_failure(str(e))
except (ConnectionRefusedError, ConnectionError) as e:
logger.warning(
"{} cannot communicate with processing node: {}".format(
self, str(e)))
except ProcessingTimeout as e:
logger.warning(
"{} timed out with error: {}. We'll try reprocessing at the next tick."
.format(self, str(e)))
# diff_id = int(os.path.basename(geojson_path).split('.')[0])
print(f'\nReading input data from the following file:\n {geojson_path}\n')
data_source = DataSource(geojson_path)
layer = data_source[0]
feature_count = len(layer)
i = 1
for feature in layer:
try:
feature_data = {
key: feature.get(value) for (key, value) in model_layer_mapping.items()
}
# Use OGRGeometry to strip z-values from polyline vertices
geometry = OGRGeometry(feature.geom.ewkt)
geometry.coord_dim = 2
feature_data['the_geom'] = geometry.ewkt
feature_data['diff_id'] = diff_id
if feature_data['type'] == 'way':
feature_data['type'] = OsmDiff.WAY
if feature_data['type'] == 'node':
feature_data['type'] = OsmDiff.NODE
if feature_data['type'] == 'relation':
feature_data['type'] = OsmDiff.RELATION
if isinstance(feature_data['tags'], str):
feature_data['tags'] = json.loads(feature_data['tags'])
if isinstance(feature_data['meta'], str):
feature_data['meta'] = json.loads(feature_data['meta'])
def test_get_pixel_value(self):
raster = GDALRaster({
'width': 5,
'height': 5,
'srid': 4326,
'bands': [{
'data': range(25)
}],
'origin': (2, 2),
'scale': (1, 1)
})
# Pixel value at origin.
point = OGRGeometry('SRID=4326;POINT(2 2)')
result = pixel_value_from_point(raster, point)
self.assertEqual(result, 0)
# Coords as tuple.
result = pixel_value_from_point(raster, (2, 2))
self.assertEqual(result, 0)
# Point in different projection.
point = OGRGeometry(
'SRID=3857;POINT(222638.9815865472 222684.20850554455)')
result = pixel_value_from_point(raster, point)
self.assertEqual(result, 0)
# Pixel value outside of raster.
point = OGRGeometry('SRID=4326;POINT(-2 2)')
result = pixel_value_from_point(raster, point)
self.assertEqual(result, None)
point = OGRGeometry('SRID=4326;POINT(8 8)')
result = pixel_value_from_point(raster, point)
self.assertEqual(result, None)
# Pixel values within the raster.
point = OGRGeometry('SRID=4326;POINT(3.5 2)')
result = pixel_value_from_point(raster, point)
self.assertEqual(result, 1)
point = OGRGeometry('SRID=4326;POINT(2 3.5)')
result = pixel_value_from_point(raster, point)
self.assertEqual(result, 5)
point = OGRGeometry('SRID=4326;POINT(6.999 6.9999)')
result = pixel_value_from_point(raster, point)
self.assertEqual(result, 24)
# Pixel value at "outer" edge of raster.
point = OGRGeometry('SRID=4326;POINT(7 7)')
result = pixel_value_from_point(raster, point)
self.assertEqual(result, 24)
# Point without srs specified.
point = OGRGeometry('POINT(2 2)')
with self.assertRaises(ValueError):
pixel_value_from_point(raster, point)
# Raster with negative scale on y axis.
raster = GDALRaster({
'width': 5,
'height': 5,
'srid': 4326,
'bands': [{
'data': range(25)
}],
'origin': (2, 2),
'scale': (1, -1)
})
point = OGRGeometry('SRID=4326;POINT(3 1)')
result = pixel_value_from_point(raster, point)
self.assertEqual(result, 6)
def test_burn_value_option(self):
geom = OGRGeometry.from_bbox((500000.0, 399800.0, 500200.0, 399900.0))
geom.srid = 3086
result = rasterize(geom, self.rast, burn_value=99)
self.assertEqual(result.bands[0].data().ravel().tolist(),
[0, 0, 99, 99])
def test09a_srs(self):
"Testing OGR Geometries with Spatial Reference objects."
for mp in self.geometries.multipolygons:
# Creating a geometry w/spatial reference
sr = SpatialReference('WGS84')
mpoly = OGRGeometry(mp.wkt, sr)
self.assertEqual(sr.wkt, mpoly.srs.wkt)
# Ensuring that SRS is propagated to clones.
klone = mpoly.clone()
self.assertEqual(sr.wkt, klone.srs.wkt)
# Ensuring all children geometries (polygons and their rings) all
# return the assigned spatial reference as well.
for poly in mpoly:
self.assertEqual(sr.wkt, poly.srs.wkt)
for ring in poly:
self.assertEqual(sr.wkt, ring.srs.wkt)
# Ensuring SRS propagate in topological ops.
a = OGRGeometry(self.geometries.topology_geoms[0].wkt_a, sr)
b = OGRGeometry(self.geometries.topology_geoms[0].wkt_b, sr)
diff = a.difference(b)
union = a.union(b)
self.assertEqual(sr.wkt, diff.srs.wkt)
self.assertEqual(sr.srid, union.srs.srid)
# Instantiating w/an integer SRID
mpoly = OGRGeometry(mp.wkt, 4326)
self.assertEqual(4326, mpoly.srid)
mpoly.srs = SpatialReference(4269)
self.assertEqual(4269, mpoly.srid)
self.assertEqual('NAD83', mpoly.srs.name)
# Incrementing through the multipolyogn after the spatial reference
# has been re-assigned.
for poly in mpoly:
self.assertEqual(mpoly.srs.wkt, poly.srs.wkt)
poly.srs = 32140
for ring in poly:
# Changing each ring in the polygon
self.assertEqual(32140, ring.srs.srid)
self.assertEqual('NAD83 / Texas South Central',
ring.srs.name)
ring.srs = str(SpatialReference(4326)) # back to WGS84
self.assertEqual(4326, ring.srs.srid)
# Using the `srid` property.
ring.srid = 4322
self.assertEqual('WGS 72', ring.srs.name)
self.assertEqual(4322, ring.srid)
def _save(feat_range=default_range, num_feat=0, num_saved=0):
if feat_range:
layer_iter = self.layer[feat_range]
else:
layer_iter = self.layer
for feat in layer_iter:
num_feat += 1
# Getting the keyword arguments
try:
kwargs = self.feature_kwargs(feat)
except LayerMapError as msg:
# Something borked the validation
if strict:
raise
elif not silent:
stream.write('Ignoring Feature ID %s because: %s\n' %
(feat.fid, msg))
else:
# Constructing the model using the keyword args
is_update = False
if self.unique:
# If we want unique models on a particular field, handle the
# geometry appropriately.
try:
# Getting the keyword arguments and retrieving
# the unique model.
u_kwargs = self.unique_kwargs(kwargs)
m = self.model.objects.using(
self.using).get(**u_kwargs)
is_update = True
# Getting the geometry (in OGR form), creating
# one from the kwargs WKT, adding in additional
# geometries, and update the attribute with the
# just-updated geometry WKT.
geom = getattr(m, self.geom_field).ogr
new = OGRGeometry(kwargs[self.geom_field])
for g in new:
geom.add(g)
setattr(m, self.geom_field, geom.wkt)
except ObjectDoesNotExist:
# No unique model exists yet, create.
m = self.model(**kwargs)
else:
m = self.model(**kwargs)
try:
# Attempting to save.
m.save(using=self.using)
num_saved += 1
if verbose:
stream.write(
'%s: %s\n' %
('Updated' if is_update else 'Saved', m))
except Exception as msg:
if strict:
# Bailing out if the `strict` keyword is set.
if not silent:
stream.write(
'Failed to save the feature (id: %s) into the '
'model with the keyword arguments:\n' %
feat.fid)
stream.write('%s\n' % kwargs)
raise
elif not silent:
stream.write(
'Failed to save %s:\n %s\nContinuing\n' %
(kwargs, msg))
# Printing progress information, if requested.
if progress and num_feat % progress_interval == 0:
stream.write('Processed %d features, saved %d ...\n' %
(num_feat, num_saved))
# Only used for status output purposes -- incremental saving uses the
# values returned here.
return num_saved, num_feat
def process(self):
"""
This method contains the logic for processing tasks asynchronously
from a background thread or from a worker. Here tasks that are
ready to be processed execute some logic. This could be communication
with a processing node or executing a pending action.
"""
try:
if self.pending_action == pending_actions.RESIZE:
resized_images = self.resize_images()
self.refresh_from_db()
self.resize_gcp(resized_images)
self.pending_action = None
self.save()
if self.auto_processing_node and not self.status in [
status_codes.FAILED, status_codes.CANCELED
]:
# No processing node assigned and need to auto assign
if self.processing_node is None:
# Assign first online node with lowest queue count
self.processing_node = ProcessingNode.find_best_available_node(
)
if self.processing_node:
self.processing_node.queue_count += 1 # Doesn't have to be accurate, it will get overridden later
self.processing_node.save()
logger.info(
"Automatically assigned processing node {} to {}".
format(self.processing_node, self))
self.save()
# Processing node assigned, but is offline and no errors
if self.processing_node and not self.processing_node.is_online(
):
# If we are queued up
# detach processing node, and reassignment
# will be processed at the next tick
if self.status == status_codes.QUEUED:
logger.info(
"Processing node {} went offline, reassigning {}..."
.format(self.processing_node, self))
self.uuid = ''
self.processing_node = None
self.status = None
self.save()
elif self.status == status_codes.RUNNING:
# Task was running and processing node went offline
# It could have crashed due to low memory
# or perhaps it went offline due to network errors.
# We can't easily differentiate between the two, so we need
# to notify the user because if it crashed due to low memory
# the user might need to take action (or be stuck in an infinite loop)
raise ProcessingError(
"Processing node went offline. This could be due to insufficient memory or a network error."
)
if self.processing_node:
# Need to process some images (UUID not yet set and task doesn't have pending actions)?
if not self.uuid and self.pending_action is None and self.status is None:
logger.info("Processing... {}".format(self))
images = [
image.path() for image in self.imageupload_set.all()
]
# Track upload progress, but limit the number of DB updates
# to every 2 seconds (and always record the 100% progress)
last_update = 0
def callback(progress):
nonlocal last_update
if time.time() - last_update >= 2 or (
progress >= 1.0 - 1e-6
and progress <= 1.0 + 1e-6):
Task.objects.filter(pk=self.id).update(
upload_progress=progress)
last_update = time.time()
# This takes a while
uuid = self.processing_node.process_new_task(
images, self.name, self.options, callback)
# Refresh task object before committing change
self.refresh_from_db()
self.uuid = uuid
self.save()
# TODO: log process has started processing
if self.pending_action is not None:
if self.pending_action == pending_actions.CANCEL:
# Do we need to cancel the task on the processing node?
logger.info("Canceling {}".format(self))
if self.processing_node and self.uuid:
# Attempt to cancel the task on the processing node
# We don't care if this fails (we tried)
try:
self.processing_node.cancel_task(self.uuid)
except ProcessingException:
logger.warning(
"Could not cancel {} on processing node. We'll proceed anyway..."
.format(self))
self.status = status_codes.CANCELED
self.pending_action = None
self.save()
else:
raise ProcessingError(
"Cannot cancel a task that has no processing node or UUID"
)
elif self.pending_action == pending_actions.RESTART:
logger.info("Restarting {}".format(self))
if self.processing_node:
# Check if the UUID is still valid, as processing nodes purge
# results after a set amount of time, the UUID might have been eliminated.
uuid_still_exists = False
if self.uuid:
try:
info = self.processing_node.get_task_info(
self.uuid)
uuid_still_exists = info['uuid'] == self.uuid
except ProcessingException:
pass
need_to_reprocess = False
if uuid_still_exists:
# Good to go
try:
self.processing_node.restart_task(
self.uuid, self.options)
except ProcessingError as e:
# Something went wrong
logger.warning(
"Could not restart {}, will start a new one"
.format(self))
need_to_reprocess = True
else:
need_to_reprocess = True
if need_to_reprocess:
logger.info(
"{} needs to be reprocessed".format(self))
# Task has been purged (or processing node is offline)
# Process this as a new task
# Removing its UUID will cause the scheduler
# to process this the next tick
self.uuid = ''
# We also remove the "rerun-from" parameter if it's set
self.options = list(
filter(lambda d: d['name'] != 'rerun-from',
self.options))
self.upload_progress = 0
self.console_output = ""
self.processing_time = -1
self.status = None
self.last_error = None
self.pending_action = None
self.running_progress = 0
self.save()
else:
raise ProcessingError(
"Cannot restart a task that has no processing node"
)
elif self.pending_action == pending_actions.REMOVE:
logger.info("Removing {}".format(self))
if self.processing_node and self.uuid:
# Attempt to delete the resources on the processing node
# We don't care if this fails, as resources on processing nodes
# Are expected to be purged on their own after a set amount of time anyway
try:
self.processing_node.remove_task(self.uuid)
except ProcessingException:
pass
# What's more important is that we delete our task properly here
self.delete()
# Stop right here!
return
if self.processing_node:
# Need to update status (first time, queued or running?)
if self.uuid and self.status in [
None, status_codes.QUEUED, status_codes.RUNNING
]:
# Update task info from processing node
info = self.processing_node.get_task_info(self.uuid)
self.processing_time = info["processingTime"]
self.status = info["status"]["code"]
current_lines_count = len(self.console_output.split("\n"))
console_output = self.processing_node.get_task_console_output(
self.uuid, current_lines_count)
if len(console_output) > 0:
self.console_output += "\n".join(console_output) + '\n'
# Update running progress
for line in console_output:
for line_match, value in self.TASK_OUTPUT_MILESTONES.items(
):
if line_match in line:
self.running_progress = value
break
if "errorMessage" in info["status"]:
self.last_error = info["status"]["errorMessage"]
# Has the task just been canceled, failed, or completed?
if self.status in [
status_codes.FAILED, status_codes.COMPLETED,
status_codes.CANCELED
]:
logger.info("Processing status: {} for {}".format(
self.status, self))
if self.status == status_codes.COMPLETED:
assets_dir = self.assets_path("")
# Remove previous assets directory
if os.path.exists(assets_dir):
logger.info(
"Removing old assets directory: {} for {}".
format(assets_dir, self))
shutil.rmtree(assets_dir)
os.makedirs(assets_dir)
logger.info(
"Downloading all.zip for {}".format(self))
# Download all assets
zip_stream = self.processing_node.download_task_asset(
self.uuid, "all.zip")
zip_path = os.path.join(assets_dir, "all.zip")
with open(zip_path, 'wb') as fd:
for chunk in zip_stream.iter_content(4096):
fd.write(chunk)
logger.info(
"Done downloading all.zip for {}".format(self))
# Extract from zip
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
logger.info(
"Extracted all.zip for {}".format(self))
# Populate *_extent fields
extent_fields = [
(os.path.realpath(
self.assets_path("odm_orthophoto",
"odm_orthophoto.tif")),
'orthophoto_extent'),
(os.path.realpath(
self.assets_path("odm_dem", "dsm.tif")),
'dsm_extent'),
(os.path.realpath(
self.assets_path("odm_dem", "dtm.tif")),
'dtm_extent'),
]
for raster_path, field in extent_fields:
if os.path.exists(raster_path):
# Read extent and SRID
raster = GDALRaster(raster_path)
extent = OGRGeometry.from_bbox(
raster.extent)
# It will be implicitly transformed into the SRID of the model’s field
# self.field = GEOSGeometry(...)
setattr(
self, field,
GEOSGeometry(extent.wkt,
srid=raster.srid))
logger.info(
"Populated extent field with {} for {}"
.format(raster_path, self))
self.update_available_assets_field()
self.running_progress = 1.0
self.save()
from app.plugins import signals as plugin_signals
plugin_signals.task_completed.send_robust(
sender=self.__class__, task_id=self.id)
else:
# FAILED, CANCELED
self.save()
else:
# Still waiting...
self.save()
except ProcessingError as e:
self.set_failure(str(e))
except (ConnectionRefusedError, ConnectionError) as e:
logger.warning(
"{} cannot communicate with processing node: {}".format(
self, str(e)))
except ProcessingTimeout as e:
logger.warning(
"{} timed out with error: {}. We'll try reprocessing at the next tick."
.format(self, str(e)))
def extract_assets_and_complete(self):
"""
Extracts assets/all.zip, populates task fields where required and assure COGs
It will raise a zipfile.BadZipFile exception is the archive is corrupted.
:return:
"""
assets_dir = self.assets_path("")
zip_path = self.assets_path("all.zip")
# Extract from zip
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
logger.info("Extracted all.zip for {}".format(self))
# Populate *_extent fields
extent_fields = [
(os.path.realpath(
self.assets_path("odm_orthophoto",
"odm_orthophoto.tif")), 'orthophoto_extent'),
(os.path.realpath(self.assets_path("odm_dem",
"dsm.tif")), 'dsm_extent'),
(os.path.realpath(self.assets_path("odm_dem",
"dtm.tif")), 'dtm_extent'),
]
for raster_path, field in extent_fields:
if os.path.exists(raster_path):
# Make sure this is a Cloud Optimized GeoTIFF
# if not, it will be created
try:
assure_cogeo(raster_path)
except IOError as e:
logger.warning(
"Cannot create Cloud Optimized GeoTIFF for %s (%s). This will result in degraded visualization performance."
% (raster_path, str(e)))
# Read extent and SRID
raster = GDALRaster(raster_path)
extent = OGRGeometry.from_bbox(raster.extent)
# Make sure PostGIS supports it
with connection.cursor() as cursor:
cursor.execute(
"SELECT SRID FROM spatial_ref_sys WHERE SRID = %s",
[raster.srid])
if cursor.rowcount == 0:
raise NodeServerError(
"Unsupported SRS {}. Please make sure you picked a supported SRS."
.format(raster.srid))
# It will be implicitly transformed into the SRID of the model’s field
# self.field = GEOSGeometry(...)
setattr(self, field, GEOSGeometry(extent.wkt,
srid=raster.srid))
logger.info("Populated extent field with {} for {}".format(
raster_path, self))
self.update_available_assets_field()
self.running_progress = 1.0
self.console_output += "Done!\n"
self.status = status_codes.COMPLETED
self.save()
from app.plugins import signals as plugin_signals
plugin_signals.task_completed.send_robust(sender=self.__class__,
task_id=self.id)
def test_empty_point_to_geos(self):
p = OGRGeometry('POINT EMPTY', srs=4326)
self.assertEqual(p.geos.ewkt, p.ewkt)
def _save(feat_range=default_range, num_feat=0, num_saved=0):
if feat_range:
layer_iter = self.layer[feat_range]
else:
layer_iter = self.layer
for feat in layer_iter:
num_feat += 1
# Getting the keyword arguments
try:
kwargs = self.feature_kwargs(feat)
except LayerMapError, msg:
# Something borked the validation
if strict: raise
elif not silent:
stream.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg))
else:
# Constructing the model using the keyword args
is_update = False
if self.unique:
# If we want unique models on a particular field, handle the
# geometry appropriately.
try:
# Getting the keyword arguments and retrieving
# the unique model.
u_kwargs = self.unique_kwargs(kwargs)
m = self.model.objects.get(**u_kwargs)
is_update = True
# Getting the geometry (in OGR form), creating
# one from the kwargs WKT, adding in additional
# geometries, and update the attribute with the
# just-updated geometry WKT.
geom = getattr(m, self.geom_field).ogr
new = OGRGeometry(kwargs[self.geom_field])
for g in new: geom.add(g)
setattr(m, self.geom_field, geom.wkt)
except ObjectDoesNotExist:
# No unique model exists yet, create.
m = self.model(**kwargs)
except AttributeError:
#there's a problem with the geometry field,
#create a new one
print 'null geometry field',
m = self.model(**kwargs)
else:
m = self.model(**kwargs)
try:
# Attempting to save.
m.save()
num_saved += 1
if verbose: stream.write('%s: %s\n' % (is_update and 'Updated' or 'Saved', m))
except SystemExit:
raise
except Exception, msg:
if self.transaction_mode == 'autocommit':
# Rolling back the transaction so that other model saves
# will work.
transaction.rollback_unless_managed()
if strict:
# Bailing out if the `strict` keyword is set.
if not silent:
stream.write('Failed to save the feature (id: %s) into the model with the keyword arguments:\n' % feat.fid)
stream.write('%s\n' % kwargs)
raise
elif not silent:
stream.write('Failed to save %s:\n %s\nContinuing\n' % (kwargs, msg))
def test_within(self):
self.assertIs(
OGRGeometry('POINT(0.5 0.5)').within(
OGRGeometry('POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))')), True)
self.assertIs(
OGRGeometry('POINT(0 0)').within(OGRGeometry('POINT(0 1)')), False)
def test_intersects(self):
self.assertIs(OGRGeometry('LINESTRING(0 0, 1 1)').intersects(OGRGeometry('LINESTRING(0 1, 1 0)')), True)
self.assertIs(OGRGeometry('LINESTRING(0 0, 0 1)').intersects(OGRGeometry('LINESTRING(1 0, 1 1)')), False)
def test_equals(self):
self.assertIs(OGRGeometry('POINT(0 0)').contains(OGRGeometry('POINT(0 0)')), True)
self.assertIs(OGRGeometry('POINT(0 0)').contains(OGRGeometry('POINT(0 1)')), False)
def test_empty(self):
self.assertIs(OGRGeometry('POINT (0 0)').empty, False)
self.assertIs(OGRGeometry('POINT EMPTY').empty, True)
def test_half_covering_geom_rasterization(self):
geom = OGRGeometry.from_bbox((500000.0, 399800.0, 500200.0, 399900.0))
geom.srid = 3086
result = rasterize(geom, self.rast)
self.assertEqual(result.bands[0].data().ravel().tolist(), [0, 0, 1, 1])
def get_geom(self, list_record):
ogrgeom = OGRGeometry(
'POINT(%s %s)' % (list_record['X_COORD'], list_record['Y_COORD']),
srs)
ogrgeom.transform(4326)
return ogrgeom.geos
def test_disjoint(self):
self.assertIs(OGRGeometry('LINESTRING(0 0, 1 1)').disjoint(OGRGeometry('LINESTRING(0 1, 1 0)')), False)
self.assertIs(OGRGeometry('LINESTRING(0 0, 0 1)').disjoint(OGRGeometry('LINESTRING(1 0, 1 1)')), True)
def test_equivalence_regression(self):
"Testing equivalence methods with non-OGRGeometry instances."
self.assertIsNotNone(OGRGeometry('POINT(0 0)'))
self.assertNotEqual(OGRGeometry('LINESTRING(0 0, 1 1)'), 3)
def geom(self):
return OGRGeometry('MULTIPOLYGON (((0 0,0.0001 0.0001,0 5,5 5,0 0)))')
def test01a_wkt(self):
"Testing WKT output."
for g in self.geometries.wkt_out:
geom = OGRGeometry(g.wkt)
self.assertEqual(g.wkt, geom.wkt)
def _get_geometry(self, val):
"""
converts geometry to geojson geom
"""
g = OGRGeometry(val)
return json.loads(g.json)
def convert_shapefile(shapefilename, srid=4674):
"""
shapefilename: considera nomenclatura de shapefile do IBGE para determinar se é UF
ou Municípios.
ex. 55UF2500GC_SIR.shp para UF e 55MU2500GC_SIR.shp para Municípios
srid: 4674 (Projeção SIRGAS 2000)
"""
# /home/nando/Desktop/IBGE/2010/55MU2500GC_SIR.shp
ds = DataSource(shapefilename)
is_uf = shapefilename.upper().find('UF') != -1
transform_coord = None
if srid != SRID:
transform_coord = CoordTransform(SpatialReference(srid),
SpatialReference(SRID))
if is_uf:
model = UF
else:
model = Municipio
ct = 0
for f in ds[0]:
# 3D para 2D se necessário
if f.geom.coord_dim != 2:
f.geom.coord_dim = 2
# converte para MultiPolygon se necessário
if isinstance(f.geom, Polygon):
g = OGRGeometry(OGRGeomType('MultiPolygon'))
g.add(f.geom)
else:
g = f.geom
# transforma coordenadas se necessário
if transform_coord:
g.transform(transform_coord)
# força 2D
g.coord_dim = 2
kwargs = {}
if is_uf:
kwargs['nome'] = capitalize_name(
unicode(f.get(CAMPO_NOME_UF), 'latin1'))
kwargs['geom'] = g.ewkt
kwargs['id_ibge'] = f.get(CAMPO_GEOCODIGO_UF)
kwargs['regiao'] = capitalize_name(
unicode(f.get(CAMPO_REGIAO_UF), 'latin1'))
kwargs['uf'] = UF_SIGLAS_DICT.get(kwargs['id_ibge'])
else:
kwargs['nome'] = capitalize_name(
unicode(f.get(CAMPO_NOME_MU), 'latin1'))
kwargs['geom'] = g.ewkt
kwargs['id_ibge'] = f.get(CAMPO_GEOCODIGO_MU)
kwargs['uf'] = UF.objects.get(pk=f.get(CAMPO_GEOCODIGO_MU)[:2])
kwargs['uf_sigla'] = kwargs['uf'].uf
kwargs['nome_abreviado'] = slugify(kwargs['nome'])
# tenta corrigir nomes duplicados, são em torno de 242 nomes repetidos
# adicionando a sigla do estado no final
if Municipio.objects.filter(
nome_abreviado=kwargs['nome_abreviado']).count() > 0:
kwargs['nome_abreviado'] = u'%s-%s' % (
kwargs['nome_abreviado'], kwargs['uf_sigla'].lower())
instance = model(**kwargs)
instance.save()
ct += 1
print(ct,
(is_uf and "Unidades Federativas criadas" or "Municipios criados"))
def test01b_gml(self):
"Testing GML output."
for g in self.geometries.wkt_out:
geom = OGRGeometry(g.wkt)
self.assertEqual(g.gml, geom.gml)
def get_extent_from_text(points, srid_in, srid_out):
"""Transform an extent from srid_in to srid_out."""
proj_in = SpatialReference(srid_in)
proj_out = SpatialReference(srid_out)
if srid_out == 900913:
if int(float(points[0])) == -180:
points[0] = -179
if int(float(points[1])) == -90:
points[1] = -89
if int(float(points[2])) == 180:
points[2] = 179
if int(float(points[3])) == 90:
points[3] = 89
wkt = 'POINT(%f %f)' % (float(points[0]), float(points[1]))
wkt2 = 'POINT(%f %f)' % (float(points[2]), float(points[3]))
ogr = OGRGeometry(wkt)
ogr2 = OGRGeometry(wkt2)
if hasattr(ogr, 'srs'):
ogr.srs = proj_in
ogr2.srs = proj_in
else:
ogr.set_srs(proj_in)
ogr2.set_srs(proj_in)
ogr.transform_to(proj_out)
ogr2.transform_to(proj_out)
wkt = ogr.wkt
wkt2 = ogr2.wkt
mins = wkt.replace('POINT (', '').replace(')', '').split(' ')
maxs = wkt2.replace('POINT (', '').replace(')', '').split(' ')
mins.append(maxs[0])
mins.append(maxs[1])
return mins
def write_with_ctypes_bindings(self, tmp_name, queryset, geo_field):
""" Scrive un file in un formato geografico da un geoqueryset; questa
funzione usa le librerie Python di GeoDjangos.
"""
# Get the shapefile driver
dr = Driver(self.out_format)
# Creating the datasource
ds = lgdal.OGR_Dr_CreateDataSource(dr._ptr, tmp_name, None)
if ds is None:
raise Exception('Could not create file!')
# Get the right geometry type number for ogr
if hasattr(geo_field, 'geom_type'):
ogr_type = OGRGeomType(geo_field.geom_type).num
else:
ogr_type = OGRGeomType(geo_field._geom).num
# Set up the native spatial reference of geometry field using the srid
if hasattr(geo_field, 'srid'):
native_srs = SpatialReference(geo_field.srid)
else:
native_srs = SpatialReference(geo_field._srid)
if self.proj_transform:
output_srs = SpatialReference(self.proj_transform)
else:
output_srs = native_srs
# create the layer
# this is crashing python26 on osx and ubuntu
layer = lgdal.OGR_DS_CreateLayer(ds, 'lyr', output_srs._ptr, ogr_type,
None)
# Create the fields
attributes = self.get_attributes()
for field in attributes:
fld = lgdal.OGR_Fld_Create(str(field.name), 4)
added = lgdal.OGR_L_CreateField(layer, fld, 0)
check_err(added)
# Getting the Layer feature definition.
feature_def = lgdal.OGR_L_GetLayerDefn(layer)
# Loop through queryset creating features
for item in self.queryset:
feat = lgdal.OGR_F_Create(feature_def)
# For now, set all fields as strings
# TODO: catch model types and convert to ogr fields
# http://www.gdal.org/ogr/classOGRFeature.html
# OGR_F_SetFieldDouble
#OFTReal => FloatField DecimalField
# OGR_F_SetFieldInteger
#OFTInteger => IntegerField
#OGR_F_SetFieldStrin
#OFTString => CharField
# OGR_F_SetFieldDateTime()
#OFTDateTime => DateTimeField
#OFTDate => TimeField
#OFTDate => DateField
for idx, field in enumerate(attributes):
value = getattr(item, field.name)
try:
string_value = str(value)
except UnicodeEncodeError, E:
# pass for now....
# http://trac.osgeo.org/gdal/ticket/882
string_value = ''
lgdal.OGR_F_SetFieldString(feat, idx, string_value)
# Transforming & setting the geometry
geom = getattr(item, geo_field.name)
# if requested we transform the input geometry
# to match the shapefiles projection 'to-be'
if geom:
ogr_geom = OGRGeometry(geom.wkt, output_srs)
if self.proj_transform:
ct = CoordTransform(native_srs, output_srs)
ogr_geom.transform(ct)
# create the geometry
check_err(lgdal.OGR_F_SetGeometry(feat, ogr_geom._ptr))
else:
# Case where geometry object is not found because of null
# value for field effectively looses whole record in shapefile
# if geometry does not exist
pass
# creat the feature in the layer.
check_err(lgdal.OGR_L_SetFeature(layer, feat))
def test14_add(self):
"Testing GeometryCollection.add()."
# Can't insert a Point into a MultiPolygon.
mp = OGRGeometry('MultiPolygon')
pnt = OGRGeometry('POINT(5 23)')
self.assertRaises(OGRException, mp.add, pnt)
# GeometryCollection.add may take an OGRGeometry (if another collection
# of the same type all child geoms will be added individually) or WKT.
for mp in self.geometries.multipolygons:
mpoly = OGRGeometry(mp.wkt)
mp1 = OGRGeometry('MultiPolygon')
mp2 = OGRGeometry('MultiPolygon')
mp3 = OGRGeometry('MultiPolygon')
for poly in mpoly:
mp1.add(poly) # Adding a geometry at a time
mp2.add(poly.wkt) # Adding WKT
mp3.add(mpoly) # Adding a MultiPolygon's entire contents at once.
for tmp in (mp1, mp2, mp3):
self.assertEqual(mpoly, tmp)
def add_boundaries_for_layer(self, config, layer, bset, options):
# Get spatial reference system for the postgis geometry field
geometry_field = Boundary._meta.get_field_by_name(GEOMETRY_COLUMN)[0]
SpatialRefSys = connections[options["database"]].ops.spatial_ref_sys()
db_srs = SpatialRefSys.objects.using(options["database"]).get(srid=geometry_field.srid).srs
if 'srid' in config and config['srid']:
layer_srs = SpatialRefSys.objects.get(srid=config['srid']).srs
else:
layer_srs = layer.srs
# Create a convertor to turn the source data into
transformer = CoordTransform(layer_srs, db_srs)
for feature in layer:
geometry = feature.geom
feature = UnicodeFeature(feature, encoding=config.get('encoding', 'ascii'))
feature.layer = layer # add additional attribute so definition file can trace back to filename
if not config.get('is_valid_func', lambda feature : True)(feature):
continue
# Transform the geometry to the correct SRS
geometry = self.polygon_to_multipolygon(geometry)
geometry.transform(transformer)
# Create simplified geometry field by collapsing points within 1/1000th of a degree.
# Since Chicago is at approx. 42 degrees latitude this works out to an margin of
# roughly 80 meters latitude and 112 meters longitude.
# Preserve topology prevents a shape from ever crossing over itself.
simple_geometry = geometry.geos.simplify(app_settings.SIMPLE_SHAPE_TOLERANCE, preserve_topology=True)
# Conversion may force multipolygons back to being polygons
simple_geometry = self.polygon_to_multipolygon(simple_geometry.ogr)
# Extract metadata into a dictionary
metadata = dict(
( (field, feature.get(field)) for field in layer.fields )
)
external_id = str(config['id_func'](feature))
feature_name = config['name_func'](feature)
feature_slug = slugify(config['slug_func'](feature).replace(u'—', '-'))
log.info('%s...' % feature_slug)
if options["merge"]:
try:
b0 = Boundary.objects.get(set=bset, slug=feature_slug)
g = OGRGeometry(OGRGeomType('MultiPolygon'))
for p in b0.shape: g.add(p.ogr)
for p in geometry: g.add(p)
b0.shape = g.wkt
if options["merge"] == "union":
# take a union of the shapes
g = self.polygon_to_multipolygon(b0.shape.cascaded_union.ogr)
b0.shape = g.wkt
# re-create the simple_shape by simplifying the union
b0.simple_shape = self.polygon_to_multipolygon(g.geos.simplify(app_settings.SIMPLE_SHAPE_TOLERANCE, preserve_topology=True).ogr).wkt
elif options["merge"] == "combine":
# extend the previous simple_shape with the new simple_shape
g = OGRGeometry(OGRGeomType('MultiPolygon'))
for p in b0.simple_shape: g.add(p.ogr)
for p in simple_geometry: g.add(p)
b0.simple_shape = g.wkt
else:
raise ValueError("Invalid value for merge option.")
b0.centroid = b0.shape.centroid
b0.extent = b0.shape.extent
b0.save()
continue
except Boundary.DoesNotExist:
pass
bdry = Boundary.objects.create(
set=bset,
set_name=bset.singular,
external_id=external_id,
name=feature_name,
slug=feature_slug,
metadata=metadata,
shape=geometry.wkt,
simple_shape=simple_geometry.wkt,
centroid=geometry.geos.centroid,
extent=geometry.extent,
label_point=config.get("label_point_func", lambda x : None)(feature)
)
if bset.extent[0] == None or bdry.extent[0] < bset.extent[0]: bset.extent[0] = bdry.extent[0]
if bset.extent[1] == None or bdry.extent[1] < bset.extent[1]: bset.extent[1] = bdry.extent[1]
if bset.extent[2] == None or bdry.extent[2] > bset.extent[2]: bset.extent[2] = bdry.extent[2]
if bset.extent[3] == None or bdry.extent[3] > bset.extent[3]: bset.extent[3] = bdry.extent[3]
def test19_equivalence_regression(self):
"Testing equivalence methods with non-OGRGeometry instances."
self.assertNotEqual(None, OGRGeometry('POINT(0 0)'))
self.assertEqual(False, OGRGeometry('LINESTRING(0 0, 1 1)') == 3)
def load_osm_diff(diff_id, logger=None):
start_time = datetime.datetime.now()
if not logger:
logger = initialize_logger()
logger.info(f'Executing script: {os.path.abspath(__file__)}')
logger.info(f'Execution start at: {start_time}')
model_layer_mapping = {
'osm_id': 'id',
'tags': 'tags',
'meta': 'meta',
'tainted': 'tainted',
'type': 'type',
}
geojson_path = r'D:\OpenStreetMap\diff_data\{diff_id}\{diff_id}.geojson'.format(
diff_id=diff_id)
logger.debug(f'Input geoJSON file: {geojson_path}')
try:
data_source = DataSource(geojson_path)
layer = data_source[0]
feature_count = len(layer)
i = 1
for feature in layer:
try:
feature_data = {
key: feature.get(value)
for (key, value) in model_layer_mapping.items()
}
# Use OGRGeometry to strip z-values from polyline vertices
geometry = OGRGeometry(feature.geom.ewkt)
geometry.coord_dim = 2
feature_data['the_geom'] = geometry.ewkt
feature_data['diff_id'] = diff_id
if feature_data['type'] == 'way':
feature_data['type'] = OsmDiff.WAY
if feature_data['type'] == 'node':
feature_data['type'] = OsmDiff.NODE
if feature_data['type'] == 'relation':
feature_data['type'] = OsmDiff.RELATION
if isinstance(feature_data['tags'], str):
feature_data['tags'] = json.loads(feature_data['tags'])
if isinstance(feature_data['meta'], str):
feature_data['meta'] = json.loads(feature_data['meta'])
route = OsmDiff(**feature_data)
route.save()
if i % 10 == 0:
logger.debug(f'Processed {i} of {feature_count} features')
if i == feature_count:
logger.debug(f'Processed {i} of {feature_count} features')
i += 1
except Exception as exc:
logger.error(f'\n\nFailed on feautre {i} of {feature_count}')
logger.error(feature_data)
logger.exception(exc)
i += 1
logger.debug(
'\nCreating buffer polygons of the newly generated ways and saving to database'
)
ways = OsmDiff.objects.filter(type=OsmDiff.WAY).filter(diff_id=diff_id)
polys = [LineString(way.the_geom).buffer(0.01) for way in ways]
multi_polys = cascaded_union(polys)
logger.debug('Saving polygons to database')
feature_count = len(multi_polys)
i = 0
for poly in multi_polys:
osm_diff_buffer = OsmDiffBuffer(
diff_id=diff_id,
created_date=datetime.datetime.today(),
the_geom=poly.wkt)
osm_diff_buffer.save()
if i % 10 == 0:
logger.debug(f'Processed {i} of {feature_count} features')
if i == feature_count:
logger.debug(f'Processed {i} of {feature_count} features')
i += 1
end_time = datetime.datetime.now()
logger.info(f'Execution complete at: {end_time}')
logger.info(f'Execution run time: {end_time-start_time}')
return True
except Exception as exc:
logger.error(f'An error occurred: load_osm_diff: {diff_id}')
logger.exception(exc)
return False
