def compute_max_zoom(self): """ Set max zoom property based on rasterlayer metadata. """ # Return manual override value if provided if self.rasterlayer.max_zoom is not None: return self.rasterlayer.max_zoom if self.dataset.srs.srid == WEB_MERCATOR_SRID: # For rasters in web mercator, use the scale directly scale = abs(self.dataset.scale.x) else: # Create a line from the center of the raster to a point that is # one pixel width away from the center. xcenter = self.dataset.extent[0] + (self.dataset.extent[2] - self.dataset.extent[0]) / 2 ycenter = self.dataset.extent[1] + (self.dataset.extent[3] - self.dataset.extent[1]) / 2 linestring = 'LINESTRING({} {}, {} {})'.format( xcenter, ycenter, xcenter + self.dataset.scale.x, ycenter ) line = OGRGeometry(linestring, srs=self.dataset.srs) # Tansform the line into web mercator. line.transform(WEB_MERCATOR_SRID) # Use the lenght of the transformed line as scale. scale = line.geos.length return utils.closest_zoomlevel(scale)
def set_simple_polylines(self, target_field_name='simple_mpoly', tolerance=2, srid=4326): """ Ben Welsh method Simplifies the source polygons so they don't use so many points. Provide a tolerance score the indicates how sharply the the lines should be redrawn. Returns True if successful. """ # Fetch the source polygon source_field_name = 'geom' source = getattr(self, source_field_name) # Fetch the target polygon where the result will be saved #target_field_name = 'simple_mpoly' target = getattr(self, target_field_name) # Simplify the source simple = source.transform(srid, True).simplify(tolerance, True) # If it's a polyline, convert it to a MultiPolyline if simple.geom_type == 'LineString': mp = OGRGeometry(OGRGeomType('MultiLineString')) mp.add(simple.wkt) target = mp.wkt # Otherwise just save out right away else: target = simple.wkt # Set the attribute setattr(self, target_field_name, target) # Save out self.save() return True
def get_wkt(value, srid=DEFAULT_PROJ): """ `value` is either a WKT string or a geometry field. Returns WKT in the projection for the given SRID. """ ogr = None if value: if isinstance(value, OGRGeometry): ogr = value elif isinstance(value, GEOSGeometry): ogr = value.ogr elif isinstance(value, basestring): match = ewkt_re.match(value) if match: ogr = OGRGeometry(match.group('wkt'), match.group('srid')) else: ogr = OGRGeometry(value) wkt = '' if ogr: # Workaround for Django bug #12312. GEOSGeometry types don't support 3D wkt; # OGRGeometry types output 3D for linestrings even if they should do 2D, causing # IntegrityError's. if ogr.dimension == 2: geos = ogr.geos geos.transform(srid) wkt = geos.wkt else: ogr.transform(srid) wkt = ogr.wkt return wkt
def region_shapefile(filename, year): '''Import the regions''' census = models.Census.objects.get_or_create(year=year)[0] ds = DataSource(filename) lyr = ds[0] spatial_backend = connections[router.db_for_write(models.Meshblock)].ops SpatialRefSys = spatial_backend.spatial_ref_sys() target_srs = SpatialRefSys.objects.get(srid=models.Region._meta.get_field_by_name('area')[0].srid).srs transform = CoordTransform(lyr.srs, target_srs) for mb_item in lyr: with transaction.atomic(): region = models.Region(census=census) g = OGRGeometry(OGRGeomType('MultiPolygon')) g.add(mb_item.geom) g.transform(transform) mpg = MultiPolygon([Polygon(*[[(x, y) for (x, y, z) in inner] for inner in middle], srid=lyr.srs.srid) for middle in g.coords], srid=lyr.srs.srid) region.area = mpg.ewkt region.region_name = mb_item['REGC2013_N'] region.save()
def extent(self, srid=WEB_MERCATOR_SRID): """ Returns bbox for layer. """ if not self._bbox or self._bbox_srid != srid: # Get bbox for raster in original coordinates meta = self.metadata xmin = meta.uperleftx ymax = meta.uperlefty xmax = xmin + meta.width * meta.scalex ymin = ymax + meta.height * meta.scaley # Create Polygon box geom = OGRGeometry(Envelope((xmin, ymin, xmax, ymax)).wkt) # Set original srs if meta.srs_wkt: geom.srs = SpatialReference(meta.srs_wkt) else: geom.srid = meta.srid # Transform to requested srid geom.transform(srid) # Calculate value range for bbox coords = geom.coords[0] xvals = [x[0] for x in coords] yvals = [x[1] for x in coords] # Set bbox self._bbox = (min(xvals), min(yvals), max(xvals), max(yvals)) self._bbox_srid = srid return self._bbox
def verify_geom(self, geom, model_field): """ Verifies the geometry -- will construct and return a GeometryCollection if necessary (for example if the model field is MultiPolygonField while the mapped shapefile only contains Polygons). """ # Downgrade a 3D geom to a 2D one, if necessary. if self.coord_dim != geom.coord_dim: geom.coord_dim = self.coord_dim if self.make_multi(geom.geom_type, model_field): # Constructing a multi-geometry type to contain the single geometry multi_type = self.MULTI_TYPES[geom.geom_type.num] g = OGRGeometry(multi_type) g.add(geom) else: g = geom # Transforming the geometry with our Coordinate Transformation object, # but only if the class variable `transform` is set w/a CoordTransform # object. if self.transform: g.transform(self.transform) # Returning the WKT of the geometry. return g.wkt
def _get_multipolygon_geometry_from_row(self, row): if row.geom_type.django == 'PolygonField': geom = OGRGeometry(OGRGeomType('MultiPolygon')) geom.add(row.geom) return geom elif row.geom_type.django == 'MultiPolygonField': return geom
def handle(self, *args, **kwargs): shape_file = args[0] ds = DataSource(shape_file) layer = ds[0] for feat in layer: try: area_id = feat['ID_0'].value except: area_id = feat['GADMID'].value area_name = unicode(feat['NAME_ENGLI'].value, 'iso-8859-1') area_varname = unicode(feat['NAME_LOCAL'].value, 'iso-8859-1') mpgeom = OGRGeometry('MultiPolygon') mpgeom.add(feat.geom) area_geom = GEOSGeometry(mpgeom.wkt) print "%s %s %s" % (area_id, area_name, area_varname) area = Area() area.shape_id = area_id area.name = area_name area.varname = area_varname area.type = 'Country' area.save() area.update_path() area.save() print "Tree ID: %d" % (area.tree_id,) areageom = Geom(area=area) areageom.geom = area_geom areageom.save()
def area_shapefile(filename, year): """Import the area units""" census = models.Census.objects.get_or_create(year=year)[0] ds = DataSource(filename) lyr = ds[0] spatial_backend = connections[router.db_for_write(models.Meshblock)].ops SpatialRefSys = spatial_backend.spatial_ref_sys() target_srs = SpatialRefSys.objects.get(srid=models.Area._meta.get_field_by_name("area")[0].srid).srs transform = CoordTransform(lyr.srs, target_srs) for mb_item in lyr: with transaction.atomic(): area = models.Area(census=census) g = OGRGeometry(OGRGeomType("MultiPolygon")) g.add(mb_item.geom) g.transform(transform) mpg = MultiPolygon( [ Polygon(*[[(x, y) for (x, y, z) in inner] for inner in middle], srid=lyr.srs.srid) for middle in g.coords ], srid=lyr.srs.srid, ) area.area = mpg.ewkt area.area_name = mb_item["AU2013_NAM"] area.save()
def transform_rd(point): ''' Please note this returns an OGRGeometry, not a point ''' src_string = '+proj=sterea +lat_0=52.15616055555555 +lon_0=5.38763888888889 +k=0.9999079 +x_0=155000 +y_0=463000 +ellps=bessel +units=m +towgs84=565.2369,50.0087,465.658,-0.406857330322398,0.350732676542563,-1.8703473836068,4.0812 +no_defs no_defs <>' src_srs = SpatialReference(src_string) geom = OGRGeometry(point.wkt, src_srs) geom.transform(4326) return geom
def test07b_closepolygons(self): "Testing closing Polygon objects." # Both rings in this geometry are not closed. poly = OGRGeometry("POLYGON((0 0, 5 0, 5 5, 0 5), (1 1, 2 1, 2 2, 2 1))") self.assertEqual(8, poly.point_count) print "\nBEGIN - expecting IllegalArgumentException; safe to ignore.\n" try: c = poly.centroid except OGRException: # Should raise an OGR exception, rings are not closed pass else: self.fail("Should have raised an OGRException!") print "\nEND - expecting IllegalArgumentException; safe to ignore.\n" # Closing the rings -- doesn't work on GDAL versions 1.4.1 and below: # http://trac.osgeo.org/gdal/ticket/1673 major, minor1, minor2 = gdal_version().split(".") if major == "1": iminor1 = int(minor1) if iminor1 < 4 or (iminor1 == 4 and minor2.startswith("1")): return poly.close_rings() self.assertEqual(10, poly.point_count) # Two closing points should've been added self.assertEqual(OGRGeometry("POINT(2.5 2.5)"), poly.centroid)
def transform(self, ct, clone=False): """ Requires GDAL. Transforms the geometry according to the given transformation object, which may be an integer SRID, and WKT or PROJ.4 string. By default, the geometry is transformed in-place and nothing is returned. However if the `clone` keyword is set, then this geometry will not be modified and a transformed clone will be returned instead. """ srid = self.srid if HAS_GDAL and srid: g = OGRGeometry(self.wkb, srid) g.transform(ct) wkb = str(g.wkb) ptr = from_wkb(wkb, len(wkb)) if clone: # User wants a cloned transformed geometry returned. return GEOSGeometry(ptr, srid=g.srid) if ptr: # Reassigning pointer, and performing post-initialization setup # again due to the reassignment. destroy_geom(self.ptr) self._ptr = ptr self._post_init(g.srid) else: raise GEOSException('Transformed WKB was invalid.')
def convOSM(wkt): """ Converts standard merkartor to osm projection as tuple """ obj = OGRGeometry(wkt) obj.srs = 'EPSG:4326' obj.transform_to(SpatialReference('EPSG:900913')) #obj.transform_to(SpatialReference('EPSG:4326')) return (obj.x, obj.y)
def set_simple_linestrings(self, tolerance=500): """ Simplifies the source linestrings so they don't use so many points. Provide a tolerance score the indicates how sharply the the lines should be redrawn. Returns True if successful. """ # Get the list of SRIDs we need to update srid_list = self.get_srid_list() # Loop through each for srid in srid_list: # Fetch the source polygon source_field_name = 'linestring_%s' % str(srid) source = getattr(self, source_field_name) # Fetch the target polygon where the result will be saved target_field_name = 'simple_%s' % source_field_name # If there's nothing to transform, drop out now. if not source: setattr(self, target_field_name, None) continue if srid != 900913: # Transform the source out of lng/lat before the simplification copy = source.transform(900913, clone=True) else: copy = deepcopy(source) # Simplify the source simple = copy.simplify(tolerance, True) # If the result is a polygon ... if simple.geom_type == 'LineString': # Create a new Multipolygon shell ml = OGRGeometry(OGRGeomType('MultiLineString')) # Transform the new poly back to its SRID simple.transform(srid) # Stuff it in the shell ml.add(simple.wkt) # Grab the WKT target = ml.wkt # If it's not a polygon... else: # It should be ready to go, so transform simple.transform(srid) # And grab the WKT target = simple.wkt # Stuff the WKT into the field setattr(self, target_field_name, target) return True
def merge_geometries(geometries_str, sep='$'): """Take a list of geometries in a string, and merge it.""" geometries = geometries_str.split(sep) if len(geometries) == 1: return geometries_str else: pool = OGRGeometry(geometries[0]) for geom in geometries: pool = pool.union(OGRGeometry(geom)) return pool.wkt
def test_srs(self): "Testing OGR Geometries with Spatial Reference objects." for mp in self.geometries.multipolygons: # Creating a geometry w/spatial reference sr = SpatialReference("WGS84") mpoly = OGRGeometry(mp.wkt, sr) self.assertEqual(sr.wkt, mpoly.srs.wkt) # Ensuring that SRS is propagated to clones. klone = mpoly.clone() self.assertEqual(sr.wkt, klone.srs.wkt) # Ensuring all children geometries (polygons and their rings) all # return the assigned spatial reference as well. for poly in mpoly: self.assertEqual(sr.wkt, poly.srs.wkt) for ring in poly: self.assertEqual(sr.wkt, ring.srs.wkt) # Ensuring SRS propagate in topological ops. a = OGRGeometry(self.geometries.topology_geoms[0].wkt_a, sr) b = OGRGeometry(self.geometries.topology_geoms[0].wkt_b, sr) diff = a.difference(b) union = a.union(b) self.assertEqual(sr.wkt, diff.srs.wkt) self.assertEqual(sr.srid, union.srs.srid) # Instantiating w/an integer SRID mpoly = OGRGeometry(mp.wkt, 4326) self.assertEqual(4326, mpoly.srid) mpoly.srs = SpatialReference(4269) self.assertEqual(4269, mpoly.srid) self.assertEqual("NAD83", mpoly.srs.name) # Incrementing through the multipolygon after the spatial reference # has been re-assigned. for poly in mpoly: self.assertEqual(mpoly.srs.wkt, poly.srs.wkt) poly.srs = 32140 for ring in poly: # Changing each ring in the polygon self.assertEqual(32140, ring.srs.srid) self.assertEqual("NAD83 / Texas South Central", ring.srs.name) ring.srs = str(SpatialReference(4326)) # back to WGS84 self.assertEqual(4326, ring.srs.srid) # Using the `srid` property. ring.srid = 4322 self.assertEqual("WGS 72", ring.srs.name) self.assertEqual(4322, ring.srid) # srs/srid may be assigned their own values, even when srs is None. mpoly = OGRGeometry(mp.wkt, srs=None) mpoly.srs = mpoly.srs mpoly.srid = mpoly.srid
def street_location(request): if request.method == 'GET': params = request.GET points = [] if params.has_key('str'): if params.has_key('int'): points += (get_locations_by_intersection(params['str'],params['int'])) elif params.has_key('door'): loc = get_location_by_door(params['str'],params['door']) if loc: points.append(loc[0].wkt) else: raise Http404 if params.has_key('out'): out = params['out'] if out == 'layer' and points: lpoints = [] for p in points: pgeom = OGRGeometry(p) pgeom.srs = 'EPSG:4326' pgeom.transform_to(SpatialReference('EPSG:900913')) pcoord = simplejson.loads(pgeom.json) lpoints.append({ "type": "point", "id": 'point', "name": "pipin", "geojson": pcoord, "icon": { "url": "/media/icons/info.png", "width": 32, "height": 37 } }) layer = { 'type': 'layer', 'id': 'layer-location', 'elements': lpoints, 'box_size': None } return HttpResponse(simplejson.dumps(layer), mimetype='text/json') else: raise Http404 else: path = request.get_full_path() + '&out=layer' context = RequestContext(request, {'params': params, 'layerpath':path}) return render_to_response('maap/street_location.html', context_instance=context) else: raise Http404 else: raise Http404
def test_symdifference(self): "Testing sym_difference()." for i in range(len(self.geometries.topology_geoms)): a = OGRGeometry(self.geometries.topology_geoms[i].wkt_a) b = OGRGeometry(self.geometries.topology_geoms[i].wkt_b) d1 = OGRGeometry(self.geometries.sdiff_geoms[i].wkt) d2 = a.sym_difference(b) self.assertEqual(d1, d2) self.assertEqual(d1, a ^ b) # __xor__ is symmetric difference operator a ^= b # testing __ixor__ self.assertEqual(d1, a)
def test_closepolygons(self): "Testing closing Polygon objects." # Both rings in this geometry are not closed. poly = OGRGeometry("POLYGON((0 0, 5 0, 5 5, 0 5), (1 1, 2 1, 2 2, 2 1))") self.assertEqual(8, poly.point_count) with self.assertRaises(GDALException): poly.centroid poly.close_rings() self.assertEqual(10, poly.point_count) # Two closing points should've been added self.assertEqual(OGRGeometry("POINT(2.5 2.5)"), poly.centroid)
def test_transform_dim(self): "Testing coordinate dimension is the same on transformed geometries." ls_orig = OGRGeometry("LINESTRING(-104.609 38.255)", 4326) ls_trans = OGRGeometry("LINESTRING(992385.4472045 481455.4944650)", 2774) prec = 3 ls_orig.transform(ls_trans.srs) # Making sure the coordinate dimension is still 2D. self.assertEqual(2, ls_orig.coord_dim) self.assertAlmostEqual(ls_trans.x[0], ls_orig.x[0], prec) self.assertAlmostEqual(ls_trans.y[0], ls_orig.y[0], prec)
def test_difference(self): "Testing difference()." for i in range(len(self.geometries.topology_geoms)): a = OGRGeometry(self.geometries.topology_geoms[i].wkt_a) b = OGRGeometry(self.geometries.topology_geoms[i].wkt_b) d1 = OGRGeometry(self.geometries.diff_geoms[i].wkt) d2 = a.difference(b) self.assertEqual(d1, d2) self.assertEqual(d1, a - b) # __sub__ is difference operator a -= b # testing __isub__ self.assertEqual(d1, a)
def test_union(self): "Testing union()." for i in range(len(self.geometries.topology_geoms)): a = OGRGeometry(self.geometries.topology_geoms[i].wkt_a) b = OGRGeometry(self.geometries.topology_geoms[i].wkt_b) u1 = OGRGeometry(self.geometries.union_geoms[i].wkt) u2 = a.union(b) self.assertEqual(u1, u2) self.assertEqual(u1, a | b) # __or__ is union operator a |= b # testing __ior__ self.assertEqual(u1, a)
def test_intersection(self): "Testing intersects() and intersection()." for i in range(len(self.geometries.topology_geoms)): a = OGRGeometry(self.geometries.topology_geoms[i].wkt_a) b = OGRGeometry(self.geometries.topology_geoms[i].wkt_b) i1 = OGRGeometry(self.geometries.intersect_geoms[i].wkt) self.assertTrue(a.intersects(b)) i2 = a.intersection(b) self.assertEqual(i1, i2) self.assertEqual(i1, a & b) # __and__ is intersection operator a &= b # testing __iand__ self.assertEqual(i1, a)
def geometry_to_multipolygon(geometry): """ Converts a Polygon to a MultiPolygon. """ if geometry.__class__.__name__ == 'MultiPolygon': return geometry elif geometry.__class__.__name__ == 'Polygon': multipolygon = OGRGeometry(OGRGeomType('MultiPolygon')) multipolygon.add(geometry) return multipolygon else: raise ValueError(_('The geometry is neither a Polygon nor a MultiPolygon.'))
def Get_PointGeom(d): if d['g']: X = d['X'] Y = d['Y'] Z = d['Z'] wkt = 'POINT(%f %f)'%(X,Y) # z niet ondersteund in geometry kennelijk... oPoint = OGRGeometry(wkt, SpatialReference('EPSG:28992')) oPoint.transform_to(SpatialReference('WGS84')) # transform van RD_NEW naar WGS84 PointGeom = oPoint.wkt return PointGeom else: return None
def test10_difference(self): "Testing difference()." for i in xrange(len(topology_geoms)): g_tup = topology_geoms[i] a = OGRGeometry(g_tup[0].wkt) b = OGRGeometry(g_tup[1].wkt) d1 = OGRGeometry(diff_geoms[i].wkt) d2 = a.difference(b) self.assertEqual(d1, d2) self.assertEqual(d1, a - b) # __sub__ is difference operator a -= b # testing __isub__ self.assertEqual(d1, a)
def test13_union(self): "Testing union()." for i in xrange(len(topology_geoms)): g_tup = topology_geoms[i] a = OGRGeometry(g_tup[0].wkt) b = OGRGeometry(g_tup[1].wkt) u1 = OGRGeometry(union_geoms[i].wkt) u2 = a.union(b) self.assertEqual(u1, u2) self.assertEqual(u1, a | b) # __or__ is union operator a |= b # testing __ior__ self.assertEqual(u1, a)
def test12_symdifference(self): "Testing sym_difference()." for i in xrange(len(topology_geoms)): g_tup = topology_geoms[i] a = OGRGeometry(g_tup[0].wkt) b = OGRGeometry(g_tup[1].wkt) d1 = OGRGeometry(sdiff_geoms[i].wkt) d2 = a.sym_difference(b) self.assertEqual(d1, d2) self.assertEqual(d1, a ^ b) # __xor__ is symmetric difference operator a ^= b # testing __ixor__ self.assertEqual(d1, a)
def polygon_to_multipolygon(self, geom): """ Convert polygons to multipolygons so all features are homogenous in the database. """ if geom.__class__.__name__ == 'Polygon': g = OGRGeometry(OGRGeomType('MultiPolygon')) g.add(geom) return g elif geom.__class__.__name__ == 'MultiPolygon': return geom else: raise ValueError('Geom is neither Polygon nor MultiPolygon.')
def merge(self, other): """ Creates a new MultiPolygon from the Polygons of two MultiPolygons. """ if hasattr(other, 'geometry'): other = other.geometry geometry = OGRGeometry(OGRGeomType('MultiPolygon')) for polygon in self.geometry: geometry.add(polygon) for polygon in other: geometry.add(polygon) return Geometry(geometry)
def test_touches(self): self.assertIs( OGRGeometry('POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))').touches(OGRGeometry('LINESTRING(0 2, 2 0)')), True ) self.assertIs(OGRGeometry('POINT(0 0)').touches(OGRGeometry('POINT(0 1)')), False)
def process(self): """ This method contains the logic for processing tasks asynchronously from a background thread or from the scheduler. Here tasks that are ready to be processed execute some logic. This could be communication with a processing node or executing a pending action. """ try: if self.auto_processing_node and not self.status in [ status_codes.FAILED, status_codes.CANCELED ]: # No processing node assigned and need to auto assign if self.processing_node is None: # Assign first online node with lowest queue count self.processing_node = ProcessingNode.find_best_available_node( ) if self.processing_node: self.processing_node.queue_count += 1 # Doesn't have to be accurate, it will get overriden later self.processing_node.save() logger.info( "Automatically assigned processing node {} to {}". format(self.processing_node, self)) self.save() # Processing node assigned, but is offline and no errors if self.processing_node and not self.processing_node.is_online( ): # Detach processing node, will be processed at the next tick logger.info( "Processing node {} went offline, reassigning {}...". format(self.processing_node, self)) self.uuid = '' self.processing_node = None self.save() if self.processing_node: # Need to process some images (UUID not yet set and task doesn't have pending actions)? if not self.uuid and self.pending_action is None and self.status is None: logger.info("Processing... {}".format(self)) images = [ image.path() for image in self.imageupload_set.all() ] # This takes a while uuid = self.processing_node.process_new_task( images, self.name, self.options) # Refresh task object before committing change self.refresh_from_db() self.uuid = uuid self.save() # TODO: log process has started processing if self.pending_action is not None: if self.pending_action == pending_actions.CANCEL: # Do we need to cancel the task on the processing node? logger.info("Canceling {}".format(self)) if self.processing_node and self.uuid: # Attempt to cancel the task on the processing node # We don't care if this fails (we tried) try: self.processing_node.cancel_task(self.uuid) self.status = None except ProcessingException: logger.warning( "Could not cancel {} on processing node. We'll proceed anyway..." .format(self)) self.status = status_codes.CANCELED self.pending_action = None self.save() else: raise ProcessingError( "Cannot cancel a task that has no processing node or UUID" ) elif self.pending_action == pending_actions.RESTART: logger.info("Restarting {}".format(self)) if self.processing_node: # Check if the UUID is still valid, as processing nodes purge # results after a set amount of time, the UUID might have eliminated. uuid_still_exists = False if self.uuid: try: info = self.processing_node.get_task_info( self.uuid) uuid_still_exists = info['uuid'] == self.uuid except ProcessingException: pass if uuid_still_exists: # Good to go try: self.processing_node.restart_task(self.uuid) except ProcessingError as e: # Something went wrong logger.warning( "Could not restart {}, will start a new one" .format(self)) self.uuid = '' else: # Task has been purged (or processing node is offline) # Process this as a new task # Removing its UUID will cause the scheduler # to process this the next tick self.uuid = '' self.console_output = "" self.processing_time = -1 self.status = None self.last_error = None self.pending_action = None self.save() else: raise ProcessingError( "Cannot restart a task that has no processing node" ) elif self.pending_action == pending_actions.REMOVE: logger.info("Removing {}".format(self)) if self.processing_node and self.uuid: # Attempt to delete the resources on the processing node # We don't care if this fails, as resources on processing nodes # Are expected to be purged on their own after a set amount of time anyway try: self.processing_node.remove_task(self.uuid) except ProcessingException: pass # What's more important is that we delete our task properly here self.delete() # Stop right here! return if self.processing_node: # Need to update status (first time, queued or running?) if self.uuid and self.status in [ None, status_codes.QUEUED, status_codes.RUNNING ]: # Update task info from processing node info = self.processing_node.get_task_info(self.uuid) self.processing_time = info["processingTime"] self.status = info["status"]["code"] current_lines_count = len( self.console_output.split("\n")) - 1 self.console_output += self.processing_node.get_task_console_output( self.uuid, current_lines_count) if "errorMessage" in info["status"]: self.last_error = info["status"]["errorMessage"] # Has the task just been canceled, failed, or completed? if self.status in [ status_codes.FAILED, status_codes.COMPLETED, status_codes.CANCELED ]: logger.info("Processing status: {} for {}".format( self.status, self)) if self.status == status_codes.COMPLETED: assets_dir = self.assets_path("") # Remove previous assets directory if os.path.exists(assets_dir): logger.info( "Removing old assets directory: {} for {}". format(assets_dir, self)) shutil.rmtree(assets_dir) os.makedirs(assets_dir) logger.info( "Downloading all.zip for {}".format(self)) # Download all assets zip_stream = self.processing_node.download_task_asset( self.uuid, "all.zip") zip_path = os.path.join(assets_dir, "all.zip") with open(zip_path, 'wb') as fd: for chunk in zip_stream.iter_content(4096): fd.write(chunk) logger.info( "Done downloading all.zip for {}".format(self)) # Extract from zip with zipfile.ZipFile(zip_path, "r") as zip_h: zip_h.extractall(assets_dir) logger.info( "Extracted all.zip for {}".format(self)) # Populate *_extent fields extent_fields = [ (os.path.realpath( self.assets_path("odm_orthophoto", "odm_orthophoto.tif")), 'orthophoto_extent'), (os.path.realpath( self.assets_path("odm_dem", "dsm.tif")), 'dsm_extent'), (os.path.realpath( self.assets_path("odm_dem", "dtm.tif")), 'dtm_extent'), ] for raster_path, field in extent_fields: if os.path.exists(raster_path): # Read extent and SRID raster = GDALRaster(raster_path) extent = OGRGeometry.from_bbox( raster.extent) # It will be implicitly transformed into the SRID of the model’s field # self.field = GEOSGeometry(...) setattr( self, field, GEOSGeometry(extent.wkt, srid=raster.srid)) logger.info( "Populated extent field with {} for {}" .format(raster_path, self)) self.update_available_assets_field() self.save() else: # FAILED, CANCELED self.save() else: # Still waiting... self.save() except ProcessingError as e: self.set_failure(str(e)) except (ConnectionRefusedError, ConnectionError) as e: logger.warning( "{} cannot communicate with processing node: {}".format( self, str(e))) except ProcessingTimeout as e: logger.warning( "{} timed out with error: {}. We'll try reprocessing at the next tick." .format(self, str(e)))
# diff_id = int(os.path.basename(geojson_path).split('.')[0]) print(f'\nReading input data from the following file:\n {geojson_path}\n') data_source = DataSource(geojson_path) layer = data_source[0] feature_count = len(layer) i = 1 for feature in layer: try: feature_data = { key: feature.get(value) for (key, value) in model_layer_mapping.items() } # Use OGRGeometry to strip z-values from polyline vertices geometry = OGRGeometry(feature.geom.ewkt) geometry.coord_dim = 2 feature_data['the_geom'] = geometry.ewkt feature_data['diff_id'] = diff_id if feature_data['type'] == 'way': feature_data['type'] = OsmDiff.WAY if feature_data['type'] == 'node': feature_data['type'] = OsmDiff.NODE if feature_data['type'] == 'relation': feature_data['type'] = OsmDiff.RELATION if isinstance(feature_data['tags'], str): feature_data['tags'] = json.loads(feature_data['tags']) if isinstance(feature_data['meta'], str): feature_data['meta'] = json.loads(feature_data['meta'])
def test_get_pixel_value(self): raster = GDALRaster({ 'width': 5, 'height': 5, 'srid': 4326, 'bands': [{ 'data': range(25) }], 'origin': (2, 2), 'scale': (1, 1) }) # Pixel value at origin. point = OGRGeometry('SRID=4326;POINT(2 2)') result = pixel_value_from_point(raster, point) self.assertEqual(result, 0) # Coords as tuple. result = pixel_value_from_point(raster, (2, 2)) self.assertEqual(result, 0) # Point in different projection. point = OGRGeometry( 'SRID=3857;POINT(222638.9815865472 222684.20850554455)') result = pixel_value_from_point(raster, point) self.assertEqual(result, 0) # Pixel value outside of raster. point = OGRGeometry('SRID=4326;POINT(-2 2)') result = pixel_value_from_point(raster, point) self.assertEqual(result, None) point = OGRGeometry('SRID=4326;POINT(8 8)') result = pixel_value_from_point(raster, point) self.assertEqual(result, None) # Pixel values within the raster. point = OGRGeometry('SRID=4326;POINT(3.5 2)') result = pixel_value_from_point(raster, point) self.assertEqual(result, 1) point = OGRGeometry('SRID=4326;POINT(2 3.5)') result = pixel_value_from_point(raster, point) self.assertEqual(result, 5) point = OGRGeometry('SRID=4326;POINT(6.999 6.9999)') result = pixel_value_from_point(raster, point) self.assertEqual(result, 24) # Pixel value at "outer" edge of raster. point = OGRGeometry('SRID=4326;POINT(7 7)') result = pixel_value_from_point(raster, point) self.assertEqual(result, 24) # Point without srs specified. point = OGRGeometry('POINT(2 2)') with self.assertRaises(ValueError): pixel_value_from_point(raster, point) # Raster with negative scale on y axis. raster = GDALRaster({ 'width': 5, 'height': 5, 'srid': 4326, 'bands': [{ 'data': range(25) }], 'origin': (2, 2), 'scale': (1, -1) }) point = OGRGeometry('SRID=4326;POINT(3 1)') result = pixel_value_from_point(raster, point) self.assertEqual(result, 6)
def test_burn_value_option(self): geom = OGRGeometry.from_bbox((500000.0, 399800.0, 500200.0, 399900.0)) geom.srid = 3086 result = rasterize(geom, self.rast, burn_value=99) self.assertEqual(result.bands[0].data().ravel().tolist(), [0, 0, 99, 99])
def test09a_srs(self): "Testing OGR Geometries with Spatial Reference objects." for mp in self.geometries.multipolygons: # Creating a geometry w/spatial reference sr = SpatialReference('WGS84') mpoly = OGRGeometry(mp.wkt, sr) self.assertEqual(sr.wkt, mpoly.srs.wkt) # Ensuring that SRS is propagated to clones. klone = mpoly.clone() self.assertEqual(sr.wkt, klone.srs.wkt) # Ensuring all children geometries (polygons and their rings) all # return the assigned spatial reference as well. for poly in mpoly: self.assertEqual(sr.wkt, poly.srs.wkt) for ring in poly: self.assertEqual(sr.wkt, ring.srs.wkt) # Ensuring SRS propagate in topological ops. a = OGRGeometry(self.geometries.topology_geoms[0].wkt_a, sr) b = OGRGeometry(self.geometries.topology_geoms[0].wkt_b, sr) diff = a.difference(b) union = a.union(b) self.assertEqual(sr.wkt, diff.srs.wkt) self.assertEqual(sr.srid, union.srs.srid) # Instantiating w/an integer SRID mpoly = OGRGeometry(mp.wkt, 4326) self.assertEqual(4326, mpoly.srid) mpoly.srs = SpatialReference(4269) self.assertEqual(4269, mpoly.srid) self.assertEqual('NAD83', mpoly.srs.name) # Incrementing through the multipolyogn after the spatial reference # has been re-assigned. for poly in mpoly: self.assertEqual(mpoly.srs.wkt, poly.srs.wkt) poly.srs = 32140 for ring in poly: # Changing each ring in the polygon self.assertEqual(32140, ring.srs.srid) self.assertEqual('NAD83 / Texas South Central', ring.srs.name) ring.srs = str(SpatialReference(4326)) # back to WGS84 self.assertEqual(4326, ring.srs.srid) # Using the `srid` property. ring.srid = 4322 self.assertEqual('WGS 72', ring.srs.name) self.assertEqual(4322, ring.srid)
def _save(feat_range=default_range, num_feat=0, num_saved=0): if feat_range: layer_iter = self.layer[feat_range] else: layer_iter = self.layer for feat in layer_iter: num_feat += 1 # Getting the keyword arguments try: kwargs = self.feature_kwargs(feat) except LayerMapError as msg: # Something borked the validation if strict: raise elif not silent: stream.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg)) else: # Constructing the model using the keyword args is_update = False if self.unique: # If we want unique models on a particular field, handle the # geometry appropriately. try: # Getting the keyword arguments and retrieving # the unique model. u_kwargs = self.unique_kwargs(kwargs) m = self.model.objects.using( self.using).get(**u_kwargs) is_update = True # Getting the geometry (in OGR form), creating # one from the kwargs WKT, adding in additional # geometries, and update the attribute with the # just-updated geometry WKT. geom = getattr(m, self.geom_field).ogr new = OGRGeometry(kwargs[self.geom_field]) for g in new: geom.add(g) setattr(m, self.geom_field, geom.wkt) except ObjectDoesNotExist: # No unique model exists yet, create. m = self.model(**kwargs) else: m = self.model(**kwargs) try: # Attempting to save. m.save(using=self.using) num_saved += 1 if verbose: stream.write( '%s: %s\n' % ('Updated' if is_update else 'Saved', m)) except Exception as msg: if strict: # Bailing out if the `strict` keyword is set. if not silent: stream.write( 'Failed to save the feature (id: %s) into the ' 'model with the keyword arguments:\n' % feat.fid) stream.write('%s\n' % kwargs) raise elif not silent: stream.write( 'Failed to save %s:\n %s\nContinuing\n' % (kwargs, msg)) # Printing progress information, if requested. if progress and num_feat % progress_interval == 0: stream.write('Processed %d features, saved %d ...\n' % (num_feat, num_saved)) # Only used for status output purposes -- incremental saving uses the # values returned here. return num_saved, num_feat
def process(self): """ This method contains the logic for processing tasks asynchronously from a background thread or from a worker. Here tasks that are ready to be processed execute some logic. This could be communication with a processing node or executing a pending action. """ try: if self.pending_action == pending_actions.RESIZE: resized_images = self.resize_images() self.refresh_from_db() self.resize_gcp(resized_images) self.pending_action = None self.save() if self.auto_processing_node and not self.status in [ status_codes.FAILED, status_codes.CANCELED ]: # No processing node assigned and need to auto assign if self.processing_node is None: # Assign first online node with lowest queue count self.processing_node = ProcessingNode.find_best_available_node( ) if self.processing_node: self.processing_node.queue_count += 1 # Doesn't have to be accurate, it will get overridden later self.processing_node.save() logger.info( "Automatically assigned processing node {} to {}". format(self.processing_node, self)) self.save() # Processing node assigned, but is offline and no errors if self.processing_node and not self.processing_node.is_online( ): # If we are queued up # detach processing node, and reassignment # will be processed at the next tick if self.status == status_codes.QUEUED: logger.info( "Processing node {} went offline, reassigning {}..." .format(self.processing_node, self)) self.uuid = '' self.processing_node = None self.status = None self.save() elif self.status == status_codes.RUNNING: # Task was running and processing node went offline # It could have crashed due to low memory # or perhaps it went offline due to network errors. # We can't easily differentiate between the two, so we need # to notify the user because if it crashed due to low memory # the user might need to take action (or be stuck in an infinite loop) raise ProcessingError( "Processing node went offline. This could be due to insufficient memory or a network error." ) if self.processing_node: # Need to process some images (UUID not yet set and task doesn't have pending actions)? if not self.uuid and self.pending_action is None and self.status is None: logger.info("Processing... {}".format(self)) images = [ image.path() for image in self.imageupload_set.all() ] # Track upload progress, but limit the number of DB updates # to every 2 seconds (and always record the 100% progress) last_update = 0 def callback(progress): nonlocal last_update if time.time() - last_update >= 2 or ( progress >= 1.0 - 1e-6 and progress <= 1.0 + 1e-6): Task.objects.filter(pk=self.id).update( upload_progress=progress) last_update = time.time() # This takes a while uuid = self.processing_node.process_new_task( images, self.name, self.options, callback) # Refresh task object before committing change self.refresh_from_db() self.uuid = uuid self.save() # TODO: log process has started processing if self.pending_action is not None: if self.pending_action == pending_actions.CANCEL: # Do we need to cancel the task on the processing node? logger.info("Canceling {}".format(self)) if self.processing_node and self.uuid: # Attempt to cancel the task on the processing node # We don't care if this fails (we tried) try: self.processing_node.cancel_task(self.uuid) except ProcessingException: logger.warning( "Could not cancel {} on processing node. We'll proceed anyway..." .format(self)) self.status = status_codes.CANCELED self.pending_action = None self.save() else: raise ProcessingError( "Cannot cancel a task that has no processing node or UUID" ) elif self.pending_action == pending_actions.RESTART: logger.info("Restarting {}".format(self)) if self.processing_node: # Check if the UUID is still valid, as processing nodes purge # results after a set amount of time, the UUID might have been eliminated. uuid_still_exists = False if self.uuid: try: info = self.processing_node.get_task_info( self.uuid) uuid_still_exists = info['uuid'] == self.uuid except ProcessingException: pass need_to_reprocess = False if uuid_still_exists: # Good to go try: self.processing_node.restart_task( self.uuid, self.options) except ProcessingError as e: # Something went wrong logger.warning( "Could not restart {}, will start a new one" .format(self)) need_to_reprocess = True else: need_to_reprocess = True if need_to_reprocess: logger.info( "{} needs to be reprocessed".format(self)) # Task has been purged (or processing node is offline) # Process this as a new task # Removing its UUID will cause the scheduler # to process this the next tick self.uuid = '' # We also remove the "rerun-from" parameter if it's set self.options = list( filter(lambda d: d['name'] != 'rerun-from', self.options)) self.upload_progress = 0 self.console_output = "" self.processing_time = -1 self.status = None self.last_error = None self.pending_action = None self.running_progress = 0 self.save() else: raise ProcessingError( "Cannot restart a task that has no processing node" ) elif self.pending_action == pending_actions.REMOVE: logger.info("Removing {}".format(self)) if self.processing_node and self.uuid: # Attempt to delete the resources on the processing node # We don't care if this fails, as resources on processing nodes # Are expected to be purged on their own after a set amount of time anyway try: self.processing_node.remove_task(self.uuid) except ProcessingException: pass # What's more important is that we delete our task properly here self.delete() # Stop right here! return if self.processing_node: # Need to update status (first time, queued or running?) if self.uuid and self.status in [ None, status_codes.QUEUED, status_codes.RUNNING ]: # Update task info from processing node info = self.processing_node.get_task_info(self.uuid) self.processing_time = info["processingTime"] self.status = info["status"]["code"] current_lines_count = len(self.console_output.split("\n")) console_output = self.processing_node.get_task_console_output( self.uuid, current_lines_count) if len(console_output) > 0: self.console_output += "\n".join(console_output) + '\n' # Update running progress for line in console_output: for line_match, value in self.TASK_OUTPUT_MILESTONES.items( ): if line_match in line: self.running_progress = value break if "errorMessage" in info["status"]: self.last_error = info["status"]["errorMessage"] # Has the task just been canceled, failed, or completed? if self.status in [ status_codes.FAILED, status_codes.COMPLETED, status_codes.CANCELED ]: logger.info("Processing status: {} for {}".format( self.status, self)) if self.status == status_codes.COMPLETED: assets_dir = self.assets_path("") # Remove previous assets directory if os.path.exists(assets_dir): logger.info( "Removing old assets directory: {} for {}". format(assets_dir, self)) shutil.rmtree(assets_dir) os.makedirs(assets_dir) logger.info( "Downloading all.zip for {}".format(self)) # Download all assets zip_stream = self.processing_node.download_task_asset( self.uuid, "all.zip") zip_path = os.path.join(assets_dir, "all.zip") with open(zip_path, 'wb') as fd: for chunk in zip_stream.iter_content(4096): fd.write(chunk) logger.info( "Done downloading all.zip for {}".format(self)) # Extract from zip with zipfile.ZipFile(zip_path, "r") as zip_h: zip_h.extractall(assets_dir) logger.info( "Extracted all.zip for {}".format(self)) # Populate *_extent fields extent_fields = [ (os.path.realpath( self.assets_path("odm_orthophoto", "odm_orthophoto.tif")), 'orthophoto_extent'), (os.path.realpath( self.assets_path("odm_dem", "dsm.tif")), 'dsm_extent'), (os.path.realpath( self.assets_path("odm_dem", "dtm.tif")), 'dtm_extent'), ] for raster_path, field in extent_fields: if os.path.exists(raster_path): # Read extent and SRID raster = GDALRaster(raster_path) extent = OGRGeometry.from_bbox( raster.extent) # It will be implicitly transformed into the SRID of the model’s field # self.field = GEOSGeometry(...) setattr( self, field, GEOSGeometry(extent.wkt, srid=raster.srid)) logger.info( "Populated extent field with {} for {}" .format(raster_path, self)) self.update_available_assets_field() self.running_progress = 1.0 self.save() from app.plugins import signals as plugin_signals plugin_signals.task_completed.send_robust( sender=self.__class__, task_id=self.id) else: # FAILED, CANCELED self.save() else: # Still waiting... self.save() except ProcessingError as e: self.set_failure(str(e)) except (ConnectionRefusedError, ConnectionError) as e: logger.warning( "{} cannot communicate with processing node: {}".format( self, str(e))) except ProcessingTimeout as e: logger.warning( "{} timed out with error: {}. We'll try reprocessing at the next tick." .format(self, str(e)))
def extract_assets_and_complete(self): """ Extracts assets/all.zip, populates task fields where required and assure COGs It will raise a zipfile.BadZipFile exception is the archive is corrupted. :return: """ assets_dir = self.assets_path("") zip_path = self.assets_path("all.zip") # Extract from zip with zipfile.ZipFile(zip_path, "r") as zip_h: zip_h.extractall(assets_dir) logger.info("Extracted all.zip for {}".format(self)) # Populate *_extent fields extent_fields = [ (os.path.realpath( self.assets_path("odm_orthophoto", "odm_orthophoto.tif")), 'orthophoto_extent'), (os.path.realpath(self.assets_path("odm_dem", "dsm.tif")), 'dsm_extent'), (os.path.realpath(self.assets_path("odm_dem", "dtm.tif")), 'dtm_extent'), ] for raster_path, field in extent_fields: if os.path.exists(raster_path): # Make sure this is a Cloud Optimized GeoTIFF # if not, it will be created try: assure_cogeo(raster_path) except IOError as e: logger.warning( "Cannot create Cloud Optimized GeoTIFF for %s (%s). This will result in degraded visualization performance." % (raster_path, str(e))) # Read extent and SRID raster = GDALRaster(raster_path) extent = OGRGeometry.from_bbox(raster.extent) # Make sure PostGIS supports it with connection.cursor() as cursor: cursor.execute( "SELECT SRID FROM spatial_ref_sys WHERE SRID = %s", [raster.srid]) if cursor.rowcount == 0: raise NodeServerError( "Unsupported SRS {}. Please make sure you picked a supported SRS." .format(raster.srid)) # It will be implicitly transformed into the SRID of the model’s field # self.field = GEOSGeometry(...) setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid)) logger.info("Populated extent field with {} for {}".format( raster_path, self)) self.update_available_assets_field() self.running_progress = 1.0 self.console_output += "Done!\n" self.status = status_codes.COMPLETED self.save() from app.plugins import signals as plugin_signals plugin_signals.task_completed.send_robust(sender=self.__class__, task_id=self.id)
def test_empty_point_to_geos(self): p = OGRGeometry('POINT EMPTY', srs=4326) self.assertEqual(p.geos.ewkt, p.ewkt)
def _save(feat_range=default_range, num_feat=0, num_saved=0): if feat_range: layer_iter = self.layer[feat_range] else: layer_iter = self.layer for feat in layer_iter: num_feat += 1 # Getting the keyword arguments try: kwargs = self.feature_kwargs(feat) except LayerMapError, msg: # Something borked the validation if strict: raise elif not silent: stream.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg)) else: # Constructing the model using the keyword args is_update = False if self.unique: # If we want unique models on a particular field, handle the # geometry appropriately. try: # Getting the keyword arguments and retrieving # the unique model. u_kwargs = self.unique_kwargs(kwargs) m = self.model.objects.get(**u_kwargs) is_update = True # Getting the geometry (in OGR form), creating # one from the kwargs WKT, adding in additional # geometries, and update the attribute with the # just-updated geometry WKT. geom = getattr(m, self.geom_field).ogr new = OGRGeometry(kwargs[self.geom_field]) for g in new: geom.add(g) setattr(m, self.geom_field, geom.wkt) except ObjectDoesNotExist: # No unique model exists yet, create. m = self.model(**kwargs) except AttributeError: #there's a problem with the geometry field, #create a new one print 'null geometry field', m = self.model(**kwargs) else: m = self.model(**kwargs) try: # Attempting to save. m.save() num_saved += 1 if verbose: stream.write('%s: %s\n' % (is_update and 'Updated' or 'Saved', m)) except SystemExit: raise except Exception, msg: if self.transaction_mode == 'autocommit': # Rolling back the transaction so that other model saves # will work. transaction.rollback_unless_managed() if strict: # Bailing out if the `strict` keyword is set. if not silent: stream.write('Failed to save the feature (id: %s) into the model with the keyword arguments:\n' % feat.fid) stream.write('%s\n' % kwargs) raise elif not silent: stream.write('Failed to save %s:\n %s\nContinuing\n' % (kwargs, msg))
def test_within(self): self.assertIs( OGRGeometry('POINT(0.5 0.5)').within( OGRGeometry('POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))')), True) self.assertIs( OGRGeometry('POINT(0 0)').within(OGRGeometry('POINT(0 1)')), False)
def test_intersects(self): self.assertIs(OGRGeometry('LINESTRING(0 0, 1 1)').intersects(OGRGeometry('LINESTRING(0 1, 1 0)')), True) self.assertIs(OGRGeometry('LINESTRING(0 0, 0 1)').intersects(OGRGeometry('LINESTRING(1 0, 1 1)')), False)
def test_equals(self): self.assertIs(OGRGeometry('POINT(0 0)').contains(OGRGeometry('POINT(0 0)')), True) self.assertIs(OGRGeometry('POINT(0 0)').contains(OGRGeometry('POINT(0 1)')), False)
def test_empty(self): self.assertIs(OGRGeometry('POINT (0 0)').empty, False) self.assertIs(OGRGeometry('POINT EMPTY').empty, True)
def test_half_covering_geom_rasterization(self): geom = OGRGeometry.from_bbox((500000.0, 399800.0, 500200.0, 399900.0)) geom.srid = 3086 result = rasterize(geom, self.rast) self.assertEqual(result.bands[0].data().ravel().tolist(), [0, 0, 1, 1])
def get_geom(self, list_record): ogrgeom = OGRGeometry( 'POINT(%s %s)' % (list_record['X_COORD'], list_record['Y_COORD']), srs) ogrgeom.transform(4326) return ogrgeom.geos
def test_disjoint(self): self.assertIs(OGRGeometry('LINESTRING(0 0, 1 1)').disjoint(OGRGeometry('LINESTRING(0 1, 1 0)')), False) self.assertIs(OGRGeometry('LINESTRING(0 0, 0 1)').disjoint(OGRGeometry('LINESTRING(1 0, 1 1)')), True)
def test_equivalence_regression(self): "Testing equivalence methods with non-OGRGeometry instances." self.assertIsNotNone(OGRGeometry('POINT(0 0)')) self.assertNotEqual(OGRGeometry('LINESTRING(0 0, 1 1)'), 3)
def geom(self): return OGRGeometry('MULTIPOLYGON (((0 0,0.0001 0.0001,0 5,5 5,0 0)))')
def test01a_wkt(self): "Testing WKT output." for g in self.geometries.wkt_out: geom = OGRGeometry(g.wkt) self.assertEqual(g.wkt, geom.wkt)
def _get_geometry(self, val): """ converts geometry to geojson geom """ g = OGRGeometry(val) return json.loads(g.json)
def convert_shapefile(shapefilename, srid=4674): """ shapefilename: considera nomenclatura de shapefile do IBGE para determinar se é UF ou Municípios. ex. 55UF2500GC_SIR.shp para UF e 55MU2500GC_SIR.shp para Municípios srid: 4674 (Projeção SIRGAS 2000) """ # /home/nando/Desktop/IBGE/2010/55MU2500GC_SIR.shp ds = DataSource(shapefilename) is_uf = shapefilename.upper().find('UF') != -1 transform_coord = None if srid != SRID: transform_coord = CoordTransform(SpatialReference(srid), SpatialReference(SRID)) if is_uf: model = UF else: model = Municipio ct = 0 for f in ds[0]: # 3D para 2D se necessário if f.geom.coord_dim != 2: f.geom.coord_dim = 2 # converte para MultiPolygon se necessário if isinstance(f.geom, Polygon): g = OGRGeometry(OGRGeomType('MultiPolygon')) g.add(f.geom) else: g = f.geom # transforma coordenadas se necessário if transform_coord: g.transform(transform_coord) # força 2D g.coord_dim = 2 kwargs = {} if is_uf: kwargs['nome'] = capitalize_name( unicode(f.get(CAMPO_NOME_UF), 'latin1')) kwargs['geom'] = g.ewkt kwargs['id_ibge'] = f.get(CAMPO_GEOCODIGO_UF) kwargs['regiao'] = capitalize_name( unicode(f.get(CAMPO_REGIAO_UF), 'latin1')) kwargs['uf'] = UF_SIGLAS_DICT.get(kwargs['id_ibge']) else: kwargs['nome'] = capitalize_name( unicode(f.get(CAMPO_NOME_MU), 'latin1')) kwargs['geom'] = g.ewkt kwargs['id_ibge'] = f.get(CAMPO_GEOCODIGO_MU) kwargs['uf'] = UF.objects.get(pk=f.get(CAMPO_GEOCODIGO_MU)[:2]) kwargs['uf_sigla'] = kwargs['uf'].uf kwargs['nome_abreviado'] = slugify(kwargs['nome']) # tenta corrigir nomes duplicados, são em torno de 242 nomes repetidos # adicionando a sigla do estado no final if Municipio.objects.filter( nome_abreviado=kwargs['nome_abreviado']).count() > 0: kwargs['nome_abreviado'] = u'%s-%s' % ( kwargs['nome_abreviado'], kwargs['uf_sigla'].lower()) instance = model(**kwargs) instance.save() ct += 1 print(ct, (is_uf and "Unidades Federativas criadas" or "Municipios criados"))
def test01b_gml(self): "Testing GML output." for g in self.geometries.wkt_out: geom = OGRGeometry(g.wkt) self.assertEqual(g.gml, geom.gml)
def get_extent_from_text(points, srid_in, srid_out): """Transform an extent from srid_in to srid_out.""" proj_in = SpatialReference(srid_in) proj_out = SpatialReference(srid_out) if srid_out == 900913: if int(float(points[0])) == -180: points[0] = -179 if int(float(points[1])) == -90: points[1] = -89 if int(float(points[2])) == 180: points[2] = 179 if int(float(points[3])) == 90: points[3] = 89 wkt = 'POINT(%f %f)' % (float(points[0]), float(points[1])) wkt2 = 'POINT(%f %f)' % (float(points[2]), float(points[3])) ogr = OGRGeometry(wkt) ogr2 = OGRGeometry(wkt2) if hasattr(ogr, 'srs'): ogr.srs = proj_in ogr2.srs = proj_in else: ogr.set_srs(proj_in) ogr2.set_srs(proj_in) ogr.transform_to(proj_out) ogr2.transform_to(proj_out) wkt = ogr.wkt wkt2 = ogr2.wkt mins = wkt.replace('POINT (', '').replace(')', '').split(' ') maxs = wkt2.replace('POINT (', '').replace(')', '').split(' ') mins.append(maxs[0]) mins.append(maxs[1]) return mins
def write_with_ctypes_bindings(self, tmp_name, queryset, geo_field): """ Scrive un file in un formato geografico da un geoqueryset; questa funzione usa le librerie Python di GeoDjangos. """ # Get the shapefile driver dr = Driver(self.out_format) # Creating the datasource ds = lgdal.OGR_Dr_CreateDataSource(dr._ptr, tmp_name, None) if ds is None: raise Exception('Could not create file!') # Get the right geometry type number for ogr if hasattr(geo_field, 'geom_type'): ogr_type = OGRGeomType(geo_field.geom_type).num else: ogr_type = OGRGeomType(geo_field._geom).num # Set up the native spatial reference of geometry field using the srid if hasattr(geo_field, 'srid'): native_srs = SpatialReference(geo_field.srid) else: native_srs = SpatialReference(geo_field._srid) if self.proj_transform: output_srs = SpatialReference(self.proj_transform) else: output_srs = native_srs # create the layer # this is crashing python26 on osx and ubuntu layer = lgdal.OGR_DS_CreateLayer(ds, 'lyr', output_srs._ptr, ogr_type, None) # Create the fields attributes = self.get_attributes() for field in attributes: fld = lgdal.OGR_Fld_Create(str(field.name), 4) added = lgdal.OGR_L_CreateField(layer, fld, 0) check_err(added) # Getting the Layer feature definition. feature_def = lgdal.OGR_L_GetLayerDefn(layer) # Loop through queryset creating features for item in self.queryset: feat = lgdal.OGR_F_Create(feature_def) # For now, set all fields as strings # TODO: catch model types and convert to ogr fields # http://www.gdal.org/ogr/classOGRFeature.html # OGR_F_SetFieldDouble #OFTReal => FloatField DecimalField # OGR_F_SetFieldInteger #OFTInteger => IntegerField #OGR_F_SetFieldStrin #OFTString => CharField # OGR_F_SetFieldDateTime() #OFTDateTime => DateTimeField #OFTDate => TimeField #OFTDate => DateField for idx, field in enumerate(attributes): value = getattr(item, field.name) try: string_value = str(value) except UnicodeEncodeError, E: # pass for now.... # http://trac.osgeo.org/gdal/ticket/882 string_value = '' lgdal.OGR_F_SetFieldString(feat, idx, string_value) # Transforming & setting the geometry geom = getattr(item, geo_field.name) # if requested we transform the input geometry # to match the shapefiles projection 'to-be' if geom: ogr_geom = OGRGeometry(geom.wkt, output_srs) if self.proj_transform: ct = CoordTransform(native_srs, output_srs) ogr_geom.transform(ct) # create the geometry check_err(lgdal.OGR_F_SetGeometry(feat, ogr_geom._ptr)) else: # Case where geometry object is not found because of null # value for field effectively looses whole record in shapefile # if geometry does not exist pass # creat the feature in the layer. check_err(lgdal.OGR_L_SetFeature(layer, feat))
def test14_add(self): "Testing GeometryCollection.add()." # Can't insert a Point into a MultiPolygon. mp = OGRGeometry('MultiPolygon') pnt = OGRGeometry('POINT(5 23)') self.assertRaises(OGRException, mp.add, pnt) # GeometryCollection.add may take an OGRGeometry (if another collection # of the same type all child geoms will be added individually) or WKT. for mp in self.geometries.multipolygons: mpoly = OGRGeometry(mp.wkt) mp1 = OGRGeometry('MultiPolygon') mp2 = OGRGeometry('MultiPolygon') mp3 = OGRGeometry('MultiPolygon') for poly in mpoly: mp1.add(poly) # Adding a geometry at a time mp2.add(poly.wkt) # Adding WKT mp3.add(mpoly) # Adding a MultiPolygon's entire contents at once. for tmp in (mp1, mp2, mp3): self.assertEqual(mpoly, tmp)
def add_boundaries_for_layer(self, config, layer, bset, options): # Get spatial reference system for the postgis geometry field geometry_field = Boundary._meta.get_field_by_name(GEOMETRY_COLUMN)[0] SpatialRefSys = connections[options["database"]].ops.spatial_ref_sys() db_srs = SpatialRefSys.objects.using(options["database"]).get(srid=geometry_field.srid).srs if 'srid' in config and config['srid']: layer_srs = SpatialRefSys.objects.get(srid=config['srid']).srs else: layer_srs = layer.srs # Create a convertor to turn the source data into transformer = CoordTransform(layer_srs, db_srs) for feature in layer: geometry = feature.geom feature = UnicodeFeature(feature, encoding=config.get('encoding', 'ascii')) feature.layer = layer # add additional attribute so definition file can trace back to filename if not config.get('is_valid_func', lambda feature : True)(feature): continue # Transform the geometry to the correct SRS geometry = self.polygon_to_multipolygon(geometry) geometry.transform(transformer) # Create simplified geometry field by collapsing points within 1/1000th of a degree. # Since Chicago is at approx. 42 degrees latitude this works out to an margin of # roughly 80 meters latitude and 112 meters longitude. # Preserve topology prevents a shape from ever crossing over itself. simple_geometry = geometry.geos.simplify(app_settings.SIMPLE_SHAPE_TOLERANCE, preserve_topology=True) # Conversion may force multipolygons back to being polygons simple_geometry = self.polygon_to_multipolygon(simple_geometry.ogr) # Extract metadata into a dictionary metadata = dict( ( (field, feature.get(field)) for field in layer.fields ) ) external_id = str(config['id_func'](feature)) feature_name = config['name_func'](feature) feature_slug = slugify(config['slug_func'](feature).replace(u'—', '-')) log.info('%s...' % feature_slug) if options["merge"]: try: b0 = Boundary.objects.get(set=bset, slug=feature_slug) g = OGRGeometry(OGRGeomType('MultiPolygon')) for p in b0.shape: g.add(p.ogr) for p in geometry: g.add(p) b0.shape = g.wkt if options["merge"] == "union": # take a union of the shapes g = self.polygon_to_multipolygon(b0.shape.cascaded_union.ogr) b0.shape = g.wkt # re-create the simple_shape by simplifying the union b0.simple_shape = self.polygon_to_multipolygon(g.geos.simplify(app_settings.SIMPLE_SHAPE_TOLERANCE, preserve_topology=True).ogr).wkt elif options["merge"] == "combine": # extend the previous simple_shape with the new simple_shape g = OGRGeometry(OGRGeomType('MultiPolygon')) for p in b0.simple_shape: g.add(p.ogr) for p in simple_geometry: g.add(p) b0.simple_shape = g.wkt else: raise ValueError("Invalid value for merge option.") b0.centroid = b0.shape.centroid b0.extent = b0.shape.extent b0.save() continue except Boundary.DoesNotExist: pass bdry = Boundary.objects.create( set=bset, set_name=bset.singular, external_id=external_id, name=feature_name, slug=feature_slug, metadata=metadata, shape=geometry.wkt, simple_shape=simple_geometry.wkt, centroid=geometry.geos.centroid, extent=geometry.extent, label_point=config.get("label_point_func", lambda x : None)(feature) ) if bset.extent[0] == None or bdry.extent[0] < bset.extent[0]: bset.extent[0] = bdry.extent[0] if bset.extent[1] == None or bdry.extent[1] < bset.extent[1]: bset.extent[1] = bdry.extent[1] if bset.extent[2] == None or bdry.extent[2] > bset.extent[2]: bset.extent[2] = bdry.extent[2] if bset.extent[3] == None or bdry.extent[3] > bset.extent[3]: bset.extent[3] = bdry.extent[3]
def test19_equivalence_regression(self): "Testing equivalence methods with non-OGRGeometry instances." self.assertNotEqual(None, OGRGeometry('POINT(0 0)')) self.assertEqual(False, OGRGeometry('LINESTRING(0 0, 1 1)') == 3)
def load_osm_diff(diff_id, logger=None): start_time = datetime.datetime.now() if not logger: logger = initialize_logger() logger.info(f'Executing script: {os.path.abspath(__file__)}') logger.info(f'Execution start at: {start_time}') model_layer_mapping = { 'osm_id': 'id', 'tags': 'tags', 'meta': 'meta', 'tainted': 'tainted', 'type': 'type', } geojson_path = r'D:\OpenStreetMap\diff_data\{diff_id}\{diff_id}.geojson'.format( diff_id=diff_id) logger.debug(f'Input geoJSON file: {geojson_path}') try: data_source = DataSource(geojson_path) layer = data_source[0] feature_count = len(layer) i = 1 for feature in layer: try: feature_data = { key: feature.get(value) for (key, value) in model_layer_mapping.items() } # Use OGRGeometry to strip z-values from polyline vertices geometry = OGRGeometry(feature.geom.ewkt) geometry.coord_dim = 2 feature_data['the_geom'] = geometry.ewkt feature_data['diff_id'] = diff_id if feature_data['type'] == 'way': feature_data['type'] = OsmDiff.WAY if feature_data['type'] == 'node': feature_data['type'] = OsmDiff.NODE if feature_data['type'] == 'relation': feature_data['type'] = OsmDiff.RELATION if isinstance(feature_data['tags'], str): feature_data['tags'] = json.loads(feature_data['tags']) if isinstance(feature_data['meta'], str): feature_data['meta'] = json.loads(feature_data['meta']) route = OsmDiff(**feature_data) route.save() if i % 10 == 0: logger.debug(f'Processed {i} of {feature_count} features') if i == feature_count: logger.debug(f'Processed {i} of {feature_count} features') i += 1 except Exception as exc: logger.error(f'\n\nFailed on feautre {i} of {feature_count}') logger.error(feature_data) logger.exception(exc) i += 1 logger.debug( '\nCreating buffer polygons of the newly generated ways and saving to database' ) ways = OsmDiff.objects.filter(type=OsmDiff.WAY).filter(diff_id=diff_id) polys = [LineString(way.the_geom).buffer(0.01) for way in ways] multi_polys = cascaded_union(polys) logger.debug('Saving polygons to database') feature_count = len(multi_polys) i = 0 for poly in multi_polys: osm_diff_buffer = OsmDiffBuffer( diff_id=diff_id, created_date=datetime.datetime.today(), the_geom=poly.wkt) osm_diff_buffer.save() if i % 10 == 0: logger.debug(f'Processed {i} of {feature_count} features') if i == feature_count: logger.debug(f'Processed {i} of {feature_count} features') i += 1 end_time = datetime.datetime.now() logger.info(f'Execution complete at: {end_time}') logger.info(f'Execution run time: {end_time-start_time}') return True except Exception as exc: logger.error(f'An error occurred: load_osm_diff: {diff_id}') logger.exception(exc) return False