def test06_spatial_filter(self):
"Testing the Layer.spatial_filter property."
ds = DataSource(get_ds_file('cities', 'shp'))
lyr = ds[0]
# When not set, it should be None.
self.assertEqual(None, lyr.spatial_filter)
# Must be set a/an OGRGeometry or 4-tuple.
self.assertRaises(TypeError, lyr._set_spatial_filter, 'foo')
# Setting the spatial filter with a tuple/list with the extent of
# a buffer centering around Pueblo.
self.assertRaises(ValueError, lyr._set_spatial_filter, list(range(5)))
filter_extent = (-105.609252, 37.255001, -103.609252, 39.255001)
lyr.spatial_filter = (-105.609252, 37.255001, -103.609252, 39.255001)
self.assertEqual(OGRGeometry.from_bbox(filter_extent), lyr.spatial_filter)
feats = [feat for feat in lyr]
self.assertEqual(1, len(feats))
self.assertEqual('Pueblo', feats[0].get('Name'))
# Setting the spatial filter with an OGRGeometry for buffer centering
# around Houston.
filter_geom = OGRGeometry('POLYGON((-96.363151 28.763374,-94.363151 28.763374,-94.363151 30.763374,-96.363151 30.763374,-96.363151 28.763374))')
lyr.spatial_filter = filter_geom
self.assertEqual(filter_geom, lyr.spatial_filter)
feats = [feat for feat in lyr]
self.assertEqual(1, len(feats))
self.assertEqual('Houston', feats[0].get('Name'))
# Clearing the spatial filter by setting it to None. Now
# should indicate that there are 3 features in the Layer.
lyr.spatial_filter = None
self.assertEqual(3, len(lyr))
def test_polygons(self):
"Testing Polygon objects."
# Testing `from_bbox` class method
bbox = (-180, -90, 180, 90)
p = OGRGeometry.from_bbox(bbox)
self.assertEqual(bbox, p.extent)
prev = OGRGeometry("POINT(0 0)")
for p in self.geometries.polygons:
poly = OGRGeometry(p.wkt)
self.assertEqual(3, poly.geom_type)
self.assertEqual("POLYGON", poly.geom_name)
self.assertEqual(p.n_p, poly.point_count)
self.assertEqual(p.n_i + 1, len(poly))
# Testing area & centroid.
self.assertAlmostEqual(p.area, poly.area, 9)
x, y = poly.centroid.tuple
self.assertAlmostEqual(p.centroid[0], x, 9)
self.assertAlmostEqual(p.centroid[1], y, 9)
# Testing equivalence
self.assertEqual(poly, OGRGeometry(p.wkt))
self.assertNotEqual(poly, prev)
if p.ext_ring_cs:
ring = poly[0]
self.assertEqual(p.ext_ring_cs, ring.tuple)
self.assertEqual(p.ext_ring_cs, poly[0].tuple)
self.assertEqual(len(p.ext_ring_cs), ring.point_count)
for r in poly:
self.assertEqual("LINEARRING", r.geom_name)
def test_covering_geom_rasterization(self):
geom = OGRGeometry.from_bbox(self.rast.extent)
geom.srid = 3086
result = rasterize(geom, self.rast)
self.assertEqual(result.bands[0].data().ravel().tolist(), [1, 1, 1, 1])
self.assertEqual(result.geotransform, self.rast.geotransform)
self.assertEqual(result.srs.wkt, self.rast.srs.wkt)
def test_rasterize_tiny_geom(self):
geom = OGRGeometry.from_bbox((5e5, 4e5, 5e5 + 1e-3, 4e5 + 1e-3))
geom.srid = 3086
# Test default configuration for small geometries (not all touched).
result = rasterize(geom, self.rast)
self.assertEqual(result.bands[0].data().ravel().tolist(), [0, 0, 0, 0])
# Switch the all touched option on.
result = rasterize(geom, self.rast, all_touched=True)
self.assertEqual(result.bands[0].data().ravel().tolist(), [1, 0, 0, 0])
def test_rasterize_merge_algorithm_add(self):
geom = OGRGeometry.from_bbox((500000.0, 399800.0, 500200.0, 399900.0))
geom.srid = 3086
result = rasterize(geom, self.rast)
self.assertEqual(result.bands[0].data().ravel().tolist(), [0, 0, 1, 1])
result = rasterize(geom, result, add=True)
if GDAL_VERSION < (2, 1, 0):
self.assertEqual(result.bands[0].data().ravel().tolist(), [0, 0, 1, 1])
else:
self.assertEqual(result.bands[0].data().ravel().tolist(), [0, 0, 2, 2])
def test_tile_index_range(self):
bounds = tile_bounds(43, 67, 8)
geom = OGRGeometry.from_bbox(bounds)
# With the default tolerance 0, the edging tiles are
# included.
idx = tile_index_range(geom.extent, 11)
self.assertEqual(idx[2] - idx[0], 2 ** 3)
self.assertEqual(idx[3] - idx[1], 2 ** 3)
# With a larger tolerance, the strictly overlaping tiles are included.
idx = tile_index_range(geom.extent, 11, tolerance=1e-3)
self.assertEqual(idx[2] - idx[0], 2 ** 3 - 1)
self.assertEqual(idx[3] - idx[1], 2 ** 3 - 1)
def list(self, request, aggregationlayer, x, y, z, frmt, *args, **kwargs):
# Select which agglayer to use for this tile.
lyr = get_object_or_404(AggregationLayer, pk=aggregationlayer)
# Compute tile boundary coorner coordinates.
bounds_coords = tile_bounds(int(x), int(y), int(z))
# Create a geometry with a 1% buffer around the tile. This buffered
# tile boundary will be used for clipping the geometry. The overflow
# will visually dissolve the polygons on the frontend visualization.
bounds = OGRGeometry.from_bbox(bounds_coords)
bounds.srid = WEB_MERCATOR_SRID
bounds = bounds.geos
bounds_buffer = bounds.buffer((bounds_coords[2] - bounds_coords[0]) / 100)
# Get the intersection of the aggregation areas and the tile boundary.
# use buffer to clip the aggregation area.
result = AggregationArea.objects.filter(
aggregationlayer=lyr,
geom__intersects=bounds,
).annotate(
intersection=Intersection('geom', bounds_buffer)
).only('id', 'name')
# Render intersection as vector tile in two different available formats.
if frmt == 'json':
result = ['{{"geometry": {0}, "properties": {{"id": {1}, "name": "{2}"}}}}'.format(dat.intersection.geojson, dat.id, dat.name) for dat in result]
result = ','.join(result)
result = '{"type": "FeatureCollection","features":[' + result + ']}'
return HttpResponse(result, content_type="application/json")
elif frmt == 'pbf':
features = [
{
"geometry": bytes(dat.intersection.wkb),
"properties": {
"id": dat.id,
"name": dat.name,
"attributes": dat.attributes,
},
} for dat in result
]
data = [
{
"name": lyr.name,
"features": features,
},
]
vtile = mapbox_vector_tile.encode(data, quantize_bounds=bounds_coords)
return HttpResponse(vtile, content_type='application/x-protobuf')
def transfer_existing_orthophoto_extent_values(apps, schema_editor):
Task = apps.get_model('app', 'Task')
for t in Task.objects.all():
print("Checking {}".format(t))
orthophoto_path = assets_path(t.project.id, t.id, "odm_orthophoto", "odm_orthophoto_4326.tif")
if os.path.exists(orthophoto_path):
print("Migrating {}".format(orthophoto_path))
raster = GDALRaster(orthophoto_path)
geom = OGRGeometry.from_bbox(raster.extent)
t.orthophoto_extent = GEOSGeometry(geom.wkt)
t.save()
os.remove(orthophoto_path)
def pixel_value_from_point(raster, point, band=0):
"""
Returns the pixel value for the coordinate of the input point from selected
band.
The input can be a point or tuple, if its a tuple it is assumed to be
coordinates in the reference system of the raster.
"""
if isinstance(point, (tuple, list)):
point = OGRGeometry('POINT({0} {1})'.format(*point))
point.srid = raster.srid
elif not point.srs or not raster.srs:
raise ValueError('Both the point and the raster are required to have a reference system specified.')
elif point.srs != raster.srs:
# Ensure the projection of the point is the same as of the raster.
point.transform(raster.srid)
# Return if point and raster do not touch.
bbox = OGRGeometry.from_bbox(raster.extent)
bbox.srs = raster.srs
if not point.intersects(bbox):
return
# Compute position of point relative to raster origin.
offset = (abs(raster.origin.x - point.coords[0]), abs(raster.origin.y - point.coords[1]))
# Compute pixel index value based on offset.
offset_index = [int(offset[0] / abs(raster.scale.x)), int(offset[1] / abs(raster.scale.y))]
# If the point is exactly on the boundary, the offset_index is rounded to
# a pixel index over the edge of the pixel. The index needs to be reduced
# by one pixel for those cases.
if offset_index[0] == raster.width:
offset_index[0] -= 1
if offset_index[1] == raster.height:
offset_index[1] -= 1
return raster.bands[band].data(offset=offset_index, size=(1, 1))[0, 0]
def test06_spatial_filter(self):
"Testing the Layer.spatial_filter property."
ds = DataSource(get_ds_file('cities', 'shp'))
lyr = ds[0]
# When not set, it should be None.
self.assertIsNone(lyr.spatial_filter)
# Must be set a/an OGRGeometry or 4-tuple.
with self.assertRaises(TypeError):
lyr._set_spatial_filter('foo')
# Setting the spatial filter with a tuple/list with the extent of
# a buffer centering around Pueblo.
with self.assertRaises(ValueError):
lyr._set_spatial_filter(list(range(5)))
filter_extent = (-105.609252, 37.255001, -103.609252, 39.255001)
lyr.spatial_filter = (-105.609252, 37.255001, -103.609252, 39.255001)
self.assertEqual(OGRGeometry.from_bbox(filter_extent),
lyr.spatial_filter)
feats = [feat for feat in lyr]
self.assertEqual(1, len(feats))
self.assertEqual('Pueblo', feats[0].get('Name'))
# Setting the spatial filter with an OGRGeometry for buffer centering
# around Houston.
filter_geom = OGRGeometry(
'POLYGON((-96.363151 28.763374,-94.363151 28.763374,'
'-94.363151 30.763374,-96.363151 30.763374,-96.363151 28.763374))')
lyr.spatial_filter = filter_geom
self.assertEqual(filter_geom, lyr.spatial_filter)
feats = [feat for feat in lyr]
self.assertEqual(1, len(feats))
self.assertEqual('Houston', feats[0].get('Name'))
# Clearing the spatial filter by setting it to None. Now
# should indicate that there are 3 features in the Layer.
lyr.spatial_filter = None
self.assertEqual(3, len(lyr))
def test_vector_tile_endpoint_json(self):
# Get url for a tile.
self.url = reverse('vectortiles-list', kwargs={'aggregationlayer': self.agglayer.id, 'z': 11, 'x': 552, 'y': 859, 'frmt': 'json'})
# Setup request with fromula that will multiply the rasterlayer by itself
response = self.client.get(self.url)
self.assertEqual(response.status_code, 200)
bounds = tile_bounds(552, 859, 11)
bounds = OGRGeometry.from_bbox(bounds)
bounds.srid = WEB_MERCATOR_SRID
result = json.loads(response.content.decode())
self.assertEqual(
'St Petersburg',
result['features'][0]['properties']['name'],
)
self.assertEqual(
'Coverall',
result['features'][1]['properties']['name'],
)
self.assertEqual(
[-9220429.22801057, 3228174.60948196],
result['features'][0]['geometry']['coordinates'][0][0][0],
)
def test_polygons(self):
"Testing Polygon objects."
# Testing `from_bbox` class method
bbox = (-180, -90, 180, 90)
p = OGRGeometry.from_bbox(bbox)
self.assertEqual(bbox, p.extent)
prev = OGRGeometry('POINT(0 0)')
for p in self.geometries.polygons:
poly = OGRGeometry(p.wkt)
self.assertEqual(3, poly.geom_type)
self.assertEqual('POLYGON', poly.geom_name)
self.assertEqual(p.n_p, poly.point_count)
self.assertEqual(p.n_i + 1, len(poly))
msg = 'Index out of range when accessing rings of a polygon: %s.'
with self.assertRaisesMessage(IndexError, msg % len(poly)):
poly.__getitem__(len(poly))
# Testing area & centroid.
self.assertAlmostEqual(p.area, poly.area, 9)
x, y = poly.centroid.tuple
self.assertAlmostEqual(p.centroid[0], x, 9)
self.assertAlmostEqual(p.centroid[1], y, 9)
# Testing equivalence
self.assertEqual(poly, OGRGeometry(p.wkt))
self.assertNotEqual(poly, prev)
if p.ext_ring_cs:
ring = poly[0]
self.assertEqual(p.ext_ring_cs, ring.tuple)
self.assertEqual(p.ext_ring_cs, poly[0].tuple)
self.assertEqual(len(p.ext_ring_cs), ring.point_count)
for r in poly:
self.assertEqual('LINEARRING', r.geom_name)
def test_polygons(self):
"Testing Polygon objects."
# Testing `from_bbox` class method
bbox = (-180, -90, 180, 90)
p = OGRGeometry.from_bbox(bbox)
self.assertEqual(bbox, p.extent)
prev = OGRGeometry('POINT(0 0)')
for p in self.geometries.polygons:
poly = OGRGeometry(p.wkt)
self.assertEqual(3, poly.geom_type)
self.assertEqual('POLYGON', poly.geom_name)
self.assertEqual(p.n_p, poly.point_count)
self.assertEqual(p.n_i + 1, len(poly))
msg = 'Index out of range when accessing rings of a polygon: %s.'
with self.assertRaisesMessage(IndexError, msg % len(poly)):
poly.__getitem__(len(poly))
# Testing area & centroid.
self.assertAlmostEqual(p.area, poly.area, 9)
x, y = poly.centroid.tuple
self.assertAlmostEqual(p.centroid[0], x, 9)
self.assertAlmostEqual(p.centroid[1], y, 9)
# Testing equivalence
self.assertEqual(poly, OGRGeometry(p.wkt))
self.assertNotEqual(poly, prev)
if p.ext_ring_cs:
ring = poly[0]
self.assertEqual(p.ext_ring_cs, ring.tuple)
self.assertEqual(p.ext_ring_cs, poly[0].tuple)
self.assertEqual(len(p.ext_ring_cs), ring.point_count)
for r in poly:
self.assertEqual('LINEARRING', r.geom_name)
def bbox(self):
return OGRGeometry(OGRGeometry.from_bbox(self.bounding_box.split(",")).json, srs=SpatialReference("WGS84"))
def process(self):
"""
This method contains the logic for processing tasks asynchronously
from a background thread or from the scheduler. Here tasks that are
ready to be processed execute some logic. This could be communication
with a processing node or executing a pending action.
"""
try:
if self.auto_processing_node and not self.status in [status_codes.FAILED, status_codes.CANCELED]:
# No processing node assigned and need to auto assign
if self.processing_node is None:
# Assign first online node with lowest queue count
self.processing_node = ProcessingNode.find_best_available_node()
if self.processing_node:
self.processing_node.queue_count += 1 # Doesn't have to be accurate, it will get overridden later
self.processing_node.save()
logger.info("Automatically assigned processing node {} to {}".format(self.processing_node, self))
self.save()
# Processing node assigned, but is offline and no errors
if self.processing_node and not self.processing_node.is_online():
# Detach processing node, will be processed at the next tick
logger.info("Processing node {} went offline, reassigning {}...".format(self.processing_node, self))
self.uuid = ''
self.processing_node = None
self.save()
if self.processing_node:
# Need to process some images (UUID not yet set and task doesn't have pending actions)?
if not self.uuid and self.pending_action is None and self.status is None:
logger.info("Processing... {}".format(self))
images = [image.path() for image in self.imageupload_set.all()]
# This takes a while
uuid = self.processing_node.process_new_task(images, self.name, self.options)
# Refresh task object before committing change
self.refresh_from_db()
self.uuid = uuid
self.save()
# TODO: log process has started processing
if self.pending_action is not None:
if self.pending_action == pending_actions.CANCEL:
# Do we need to cancel the task on the processing node?
logger.info("Canceling {}".format(self))
if self.processing_node and self.uuid:
# Attempt to cancel the task on the processing node
# We don't care if this fails (we tried)
try:
self.processing_node.cancel_task(self.uuid)
self.status = None
except ProcessingException:
logger.warning("Could not cancel {} on processing node. We'll proceed anyway...".format(self))
self.status = status_codes.CANCELED
self.pending_action = None
self.save()
else:
raise ProcessingError("Cannot cancel a task that has no processing node or UUID")
elif self.pending_action == pending_actions.RESTART:
logger.info("Restarting {}".format(self))
if self.processing_node:
# Check if the UUID is still valid, as processing nodes purge
# results after a set amount of time, the UUID might have eliminated.
uuid_still_exists = False
if self.uuid:
try:
info = self.processing_node.get_task_info(self.uuid)
uuid_still_exists = info['uuid'] == self.uuid
except ProcessingException:
pass
if uuid_still_exists:
# Good to go
try:
self.processing_node.restart_task(self.uuid)
except ProcessingError as e:
# Something went wrong
logger.warning("Could not restart {}, will start a new one".format(self))
self.uuid = ''
else:
# Task has been purged (or processing node is offline)
# Process this as a new task
# Removing its UUID will cause the scheduler
# to process this the next tick
self.uuid = ''
self.console_output = ""
self.processing_time = -1
self.status = None
self.last_error = None
self.pending_action = None
self.save()
else:
raise ProcessingError("Cannot restart a task that has no processing node")
elif self.pending_action == pending_actions.REMOVE:
logger.info("Removing {}".format(self))
if self.processing_node and self.uuid:
# Attempt to delete the resources on the processing node
# We don't care if this fails, as resources on processing nodes
# Are expected to be purged on their own after a set amount of time anyway
try:
self.processing_node.remove_task(self.uuid)
except ProcessingException:
pass
# What's more important is that we delete our task properly here
self.delete()
# Stop right here!
return
if self.processing_node:
# Need to update status (first time, queued or running?)
if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]:
# Update task info from processing node
info = self.processing_node.get_task_info(self.uuid)
self.processing_time = info["processingTime"]
self.status = info["status"]["code"]
current_lines_count = len(self.console_output.split("\n")) - 1
self.console_output += self.processing_node.get_task_console_output(self.uuid, current_lines_count)
if "errorMessage" in info["status"]:
self.last_error = info["status"]["errorMessage"]
# Has the task just been canceled, failed, or completed?
if self.status in [status_codes.FAILED, status_codes.COMPLETED, status_codes.CANCELED]:
logger.info("Processing status: {} for {}".format(self.status, self))
if self.status == status_codes.COMPLETED:
assets_dir = self.assets_path("")
# Remove previous assets directory
if os.path.exists(assets_dir):
logger.info("Removing old assets directory: {} for {}".format(assets_dir, self))
shutil.rmtree(assets_dir)
os.makedirs(assets_dir)
logger.info("Downloading all.zip for {}".format(self))
# Download all assets
zip_stream = self.processing_node.download_task_asset(self.uuid, "all.zip")
zip_path = os.path.join(assets_dir, "all.zip")
with open(zip_path, 'wb') as fd:
for chunk in zip_stream.iter_content(4096):
fd.write(chunk)
logger.info("Done downloading all.zip for {}".format(self))
# Extract from zip
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
logger.info("Extracted all.zip for {}".format(self))
# Populate *_extent fields
extent_fields = [
(os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")),
'orthophoto_extent'),
(os.path.realpath(self.assets_path("odm_dem", "dsm.tif")),
'dsm_extent'),
(os.path.realpath(self.assets_path("odm_dem", "dtm.tif")),
'dtm_extent'),
]
for raster_path, field in extent_fields:
if os.path.exists(raster_path):
# Read extent and SRID
raster = GDALRaster(raster_path)
extent = OGRGeometry.from_bbox(raster.extent)
# It will be implicitly transformed into the SRID of the model’s field
# self.field = GEOSGeometry(...)
setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid))
logger.info("Populated extent field with {} for {}".format(raster_path, self))
self.update_available_assets_field()
self.save()
else:
# FAILED, CANCELED
self.save()
else:
# Still waiting...
self.save()
except ProcessingError as e:
self.set_failure(str(e))
except (ConnectionRefusedError, ConnectionError) as e:
logger.warning("{} cannot communicate with processing node: {}".format(self, str(e)))
except ProcessingTimeout as e:
logger.warning("{} timed out with error: {}. We'll try reprocessing at the next tick.".format(self, str(e)))
def process(self):
"""
This method contains the logic for processing tasks asynchronously
from a background thread or from a worker. Here tasks that are
ready to be processed execute some logic. This could be communication
with a processing node or executing a pending action.
"""
try:
if self.pending_action == pending_actions.RESIZE:
resized_images = self.resize_images()
self.resize_gcp(resized_images)
self.pending_action = None
self.save()
if self.auto_processing_node and not self.status in [status_codes.FAILED, status_codes.CANCELED]:
# No processing node assigned and need to auto assign
if self.processing_node is None:
# Assign first online node with lowest queue count
self.processing_node = ProcessingNode.find_best_available_node()
if self.processing_node:
self.processing_node.queue_count += 1 # Doesn't have to be accurate, it will get overridden later
self.processing_node.save()
logger.info("Automatically assigned processing node {} to {}".format(self.processing_node, self))
self.save()
# Processing node assigned, but is offline and no errors
if self.processing_node and not self.processing_node.is_online():
# If we are queued up
# detach processing node, and reassignment
# will be processed at the next tick
if self.status == status_codes.QUEUED:
logger.info("Processing node {} went offline, reassigning {}...".format(self.processing_node, self))
self.uuid = ''
self.processing_node = None
self.status = None
self.save()
elif self.status == status_codes.RUNNING:
# Task was running and processing node went offline
# It could have crashed due to low memory
# or perhaps it went offline due to network errors.
# We can't easily differentiate between the two, so we need
# to notify the user because if it crashed due to low memory
# the user might need to take action (or be stuck in an infinite loop)
raise ProcessingError("Processing node went offline. This could be due to insufficient memory or a network error.")
if self.processing_node:
# Need to process some images (UUID not yet set and task doesn't have pending actions)?
if not self.uuid and self.pending_action is None and self.status is None:
logger.info("Processing... {}".format(self))
images = [image.path() for image in self.imageupload_set.all()]
# This takes a while
uuid = self.processing_node.process_new_task(images, self.name, self.options)
# Refresh task object before committing change
self.refresh_from_db()
self.uuid = uuid
self.save()
# TODO: log process has started processing
if self.pending_action is not None:
if self.pending_action == pending_actions.CANCEL:
# Do we need to cancel the task on the processing node?
logger.info("Canceling {}".format(self))
if self.processing_node and self.uuid:
# Attempt to cancel the task on the processing node
# We don't care if this fails (we tried)
try:
self.processing_node.cancel_task(self.uuid)
except ProcessingException:
logger.warning("Could not cancel {} on processing node. We'll proceed anyway...".format(self))
self.status = status_codes.CANCELED
self.pending_action = None
self.save()
else:
raise ProcessingError("Cannot cancel a task that has no processing node or UUID")
elif self.pending_action == pending_actions.RESTART:
logger.info("Restarting {}".format(self))
if self.processing_node:
# Check if the UUID is still valid, as processing nodes purge
# results after a set amount of time, the UUID might have been eliminated.
uuid_still_exists = False
if self.uuid:
try:
info = self.processing_node.get_task_info(self.uuid)
uuid_still_exists = info['uuid'] == self.uuid
except ProcessingException:
pass
need_to_reprocess = False
if uuid_still_exists:
# Good to go
try:
self.processing_node.restart_task(self.uuid, self.options)
except ProcessingError as e:
# Something went wrong
logger.warning("Could not restart {}, will start a new one".format(self))
need_to_reprocess = True
else:
need_to_reprocess = True
if need_to_reprocess:
logger.info("{} needs to be reprocessed".format(self))
# Task has been purged (or processing node is offline)
# Process this as a new task
# Removing its UUID will cause the scheduler
# to process this the next tick
self.uuid = ''
# We also remove the "rerun-from" parameter if it's set
self.options = list(filter(lambda d: d['name'] != 'rerun-from', self.options))
self.console_output = ""
self.processing_time = -1
self.status = None
self.last_error = None
self.pending_action = None
self.save()
else:
raise ProcessingError("Cannot restart a task that has no processing node")
elif self.pending_action == pending_actions.REMOVE:
logger.info("Removing {}".format(self))
if self.processing_node and self.uuid:
# Attempt to delete the resources on the processing node
# We don't care if this fails, as resources on processing nodes
# Are expected to be purged on their own after a set amount of time anyway
try:
self.processing_node.remove_task(self.uuid)
except ProcessingException:
pass
# What's more important is that we delete our task properly here
self.delete()
# Stop right here!
return
if self.processing_node:
# Need to update status (first time, queued or running?)
if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]:
# Update task info from processing node
info = self.processing_node.get_task_info(self.uuid)
self.processing_time = info["processingTime"]
self.status = info["status"]["code"]
current_lines_count = len(self.console_output.split("\n"))
console_output = self.processing_node.get_task_console_output(self.uuid, current_lines_count)
if len(console_output) > 0:
self.console_output += console_output + '\n'
if "errorMessage" in info["status"]:
self.last_error = info["status"]["errorMessage"]
# Has the task just been canceled, failed, or completed?
if self.status in [status_codes.FAILED, status_codes.COMPLETED, status_codes.CANCELED]:
logger.info("Processing status: {} for {}".format(self.status, self))
if self.status == status_codes.COMPLETED:
assets_dir = self.assets_path("")
# Remove previous assets directory
if os.path.exists(assets_dir):
logger.info("Removing old assets directory: {} for {}".format(assets_dir, self))
shutil.rmtree(assets_dir)
os.makedirs(assets_dir)
logger.info("Downloading all.zip for {}".format(self))
# Download all assets
zip_stream = self.processing_node.download_task_asset(self.uuid, "all.zip")
zip_path = os.path.join(assets_dir, "all.zip")
with open(zip_path, 'wb') as fd:
for chunk in zip_stream.iter_content(4096):
fd.write(chunk)
logger.info("Done downloading all.zip for {}".format(self))
# Extract from zip
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
logger.info("Extracted all.zip for {}".format(self))
# Populate *_extent fields
extent_fields = [
(os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")),
'orthophoto_extent'),
(os.path.realpath(self.assets_path("odm_dem", "dsm.tif")),
'dsm_extent'),
(os.path.realpath(self.assets_path("odm_dem", "dtm.tif")),
'dtm_extent'),
]
for raster_path, field in extent_fields:
if os.path.exists(raster_path):
# Read extent and SRID
raster = GDALRaster(raster_path)
extent = OGRGeometry.from_bbox(raster.extent)
# It will be implicitly transformed into the SRID of the model’s field
# self.field = GEOSGeometry(...)
setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid))
logger.info("Populated extent field with {} for {}".format(raster_path, self))
self.update_available_assets_field()
self.save()
from app.plugins import signals as plugin_signals
plugin_signals.task_completed.send_robust(sender=self.__class__, task_id=self.id)
else:
# FAILED, CANCELED
self.save()
else:
# Still waiting...
self.save()
except ProcessingError as e:
self.set_failure(str(e))
except (ConnectionRefusedError, ConnectionError) as e:
logger.warning("{} cannot communicate with processing node: {}".format(self, str(e)))
except ProcessingTimeout as e:
logger.warning("{} timed out with error: {}. We'll try reprocessing at the next tick.".format(self, str(e)))
def test_burn_value_option(self):
geom = OGRGeometry.from_bbox((500000.0, 399800.0, 500200.0, 399900.0))
geom.srid = 3086
result = rasterize(geom, self.rast, burn_value=99)
self.assertEqual(result.bands[0].data().ravel().tolist(),
[0, 0, 99, 99])
def test_half_covering_geom_rasterization(self):
geom = OGRGeometry.from_bbox((500000.0, 399800.0, 500200.0, 399900.0))
geom.srid = 3086
result = rasterize(geom, self.rast)
self.assertEqual(result.bands[0].data().ravel().tolist(), [0, 0, 1, 1])
def extract_assets_and_complete(self):
"""
Extracts assets/all.zip, populates task fields where required and assure COGs
It will raise a zipfile.BadZipFile exception is the archive is corrupted.
:return:
"""
assets_dir = self.assets_path("")
zip_path = self.assets_path("all.zip")
# Extract from zip
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
logger.info("Extracted all.zip for {}".format(self))
# Populate *_extent fields
extent_fields = [
(os.path.realpath(
self.assets_path("odm_orthophoto",
"odm_orthophoto.tif")), 'orthophoto_extent'),
(os.path.realpath(self.assets_path("odm_dem",
"dsm.tif")), 'dsm_extent'),
(os.path.realpath(self.assets_path("odm_dem",
"dtm.tif")), 'dtm_extent'),
]
for raster_path, field in extent_fields:
if os.path.exists(raster_path):
# Make sure this is a Cloud Optimized GeoTIFF
# if not, it will be created
try:
assure_cogeo(raster_path)
except IOError as e:
logger.warning(
"Cannot create Cloud Optimized GeoTIFF for %s (%s). This will result in degraded visualization performance."
% (raster_path, str(e)))
# Read extent and SRID
raster = GDALRaster(raster_path)
extent = OGRGeometry.from_bbox(raster.extent)
# Make sure PostGIS supports it
with connection.cursor() as cursor:
cursor.execute(
"SELECT SRID FROM spatial_ref_sys WHERE SRID = %s",
[raster.srid])
if cursor.rowcount == 0:
raise NodeServerError(
gettext(
"Unsupported SRS %(code)s. Please make sure you picked a supported SRS."
) % {'code': str(raster.srid)})
# It will be implicitly transformed into the SRID of the model’s field
# self.field = GEOSGeometry(...)
setattr(self, field, GEOSGeometry(extent.wkt,
srid=raster.srid))
logger.info("Populated extent field with {} for {}".format(
raster_path, self))
self.update_available_assets_field()
self.running_progress = 1.0
self.console_output += gettext("Done!") + "\n"
self.status = status_codes.COMPLETED
self.save()
from app.plugins import signals as plugin_signals
plugin_signals.task_completed.send_robust(sender=self.__class__,
task_id=self.id)
def test_half_covering_geom_rasterization(self):
geom = OGRGeometry.from_bbox((500000.0, 399800.0, 500200.0, 399900.0))
geom.srid = 3086
result = rasterize(geom, self.rast)
self.assertEqual(result.bands[0].data().ravel().tolist(), [0, 0, 1, 1])
def get_bbox(self, obj):
bbox = obj.bbox
geom = OGRGeometry.from_bbox([bbox[0], bbox[2], bbox[1], bbox[3]])
return json.loads(geom.json)
def process(self):
"""
This method contains the logic for processing tasks asynchronously
from a background thread or from a worker. Here tasks that are
ready to be processed execute some logic. This could be communication
with a processing node or executing a pending action.
"""
try:
if self.pending_action == pending_actions.RESIZE:
resized_images = self.resize_images()
self.refresh_from_db()
self.resize_gcp(resized_images)
self.pending_action = None
self.save()
if self.auto_processing_node and not self.status in [
status_codes.FAILED, status_codes.CANCELED
]:
# No processing node assigned and need to auto assign
if self.processing_node is None:
# Assign first online node with lowest queue count
self.processing_node = ProcessingNode.find_best_available_node(
)
if self.processing_node:
self.processing_node.queue_count += 1 # Doesn't have to be accurate, it will get overridden later
self.processing_node.save()
logger.info(
"Automatically assigned processing node {} to {}".
format(self.processing_node, self))
self.save()
# Processing node assigned, but is offline and no errors
if self.processing_node and not self.processing_node.is_online(
):
# If we are queued up
# detach processing node, and reassignment
# will be processed at the next tick
if self.status == status_codes.QUEUED:
logger.info(
"Processing node {} went offline, reassigning {}..."
.format(self.processing_node, self))
self.uuid = ''
self.processing_node = None
self.status = None
self.save()
elif self.status == status_codes.RUNNING:
# Task was running and processing node went offline
# It could have crashed due to low memory
# or perhaps it went offline due to network errors.
# We can't easily differentiate between the two, so we need
# to notify the user because if it crashed due to low memory
# the user might need to take action (or be stuck in an infinite loop)
raise ProcessingError(
"Processing node went offline. This could be due to insufficient memory or a network error."
)
if self.processing_node:
# Need to process some images (UUID not yet set and task doesn't have pending actions)?
if not self.uuid and self.pending_action is None and self.status is None:
logger.info("Processing... {}".format(self))
images = [
image.path() for image in self.imageupload_set.all()
]
# Track upload progress, but limit the number of DB updates
# to every 2 seconds (and always record the 100% progress)
last_update = 0
def callback(progress):
nonlocal last_update
if time.time() - last_update >= 2 or (
progress >= 1.0 - 1e-6
and progress <= 1.0 + 1e-6):
Task.objects.filter(pk=self.id).update(
upload_progress=progress)
last_update = time.time()
# This takes a while
uuid = self.processing_node.process_new_task(
images, self.name, self.options, callback)
# Refresh task object before committing change
self.refresh_from_db()
self.uuid = uuid
self.save()
# TODO: log process has started processing
if self.pending_action is not None:
if self.pending_action == pending_actions.CANCEL:
# Do we need to cancel the task on the processing node?
logger.info("Canceling {}".format(self))
if self.processing_node and self.uuid:
# Attempt to cancel the task on the processing node
# We don't care if this fails (we tried)
try:
self.processing_node.cancel_task(self.uuid)
except ProcessingException:
logger.warning(
"Could not cancel {} on processing node. We'll proceed anyway..."
.format(self))
self.status = status_codes.CANCELED
self.pending_action = None
self.save()
else:
raise ProcessingError(
"Cannot cancel a task that has no processing node or UUID"
)
elif self.pending_action == pending_actions.RESTART:
logger.info("Restarting {}".format(self))
if self.processing_node:
# Check if the UUID is still valid, as processing nodes purge
# results after a set amount of time, the UUID might have been eliminated.
uuid_still_exists = False
if self.uuid:
try:
info = self.processing_node.get_task_info(
self.uuid)
uuid_still_exists = info['uuid'] == self.uuid
except ProcessingException:
pass
need_to_reprocess = False
if uuid_still_exists:
# Good to go
try:
self.processing_node.restart_task(
self.uuid, self.options)
except ProcessingError as e:
# Something went wrong
logger.warning(
"Could not restart {}, will start a new one"
.format(self))
need_to_reprocess = True
else:
need_to_reprocess = True
if need_to_reprocess:
logger.info(
"{} needs to be reprocessed".format(self))
# Task has been purged (or processing node is offline)
# Process this as a new task
# Removing its UUID will cause the scheduler
# to process this the next tick
self.uuid = ''
# We also remove the "rerun-from" parameter if it's set
self.options = list(
filter(lambda d: d['name'] != 'rerun-from',
self.options))
self.upload_progress = 0
self.console_output = ""
self.processing_time = -1
self.status = None
self.last_error = None
self.pending_action = None
self.running_progress = 0
self.save()
else:
raise ProcessingError(
"Cannot restart a task that has no processing node"
)
elif self.pending_action == pending_actions.REMOVE:
logger.info("Removing {}".format(self))
if self.processing_node and self.uuid:
# Attempt to delete the resources on the processing node
# We don't care if this fails, as resources on processing nodes
# Are expected to be purged on their own after a set amount of time anyway
try:
self.processing_node.remove_task(self.uuid)
except ProcessingException:
pass
# What's more important is that we delete our task properly here
self.delete()
# Stop right here!
return
if self.processing_node:
# Need to update status (first time, queued or running?)
if self.uuid and self.status in [
None, status_codes.QUEUED, status_codes.RUNNING
]:
# Update task info from processing node
info = self.processing_node.get_task_info(self.uuid)
self.processing_time = info["processingTime"]
self.status = info["status"]["code"]
current_lines_count = len(self.console_output.split("\n"))
console_output = self.processing_node.get_task_console_output(
self.uuid, current_lines_count)
if len(console_output) > 0:
self.console_output += "\n".join(console_output) + '\n'
# Update running progress
for line in console_output:
for line_match, value in self.TASK_OUTPUT_MILESTONES.items(
):
if line_match in line:
self.running_progress = value
break
if "errorMessage" in info["status"]:
self.last_error = info["status"]["errorMessage"]
# Has the task just been canceled, failed, or completed?
if self.status in [
status_codes.FAILED, status_codes.COMPLETED,
status_codes.CANCELED
]:
logger.info("Processing status: {} for {}".format(
self.status, self))
if self.status == status_codes.COMPLETED:
assets_dir = self.assets_path("")
# Remove previous assets directory
if os.path.exists(assets_dir):
logger.info(
"Removing old assets directory: {} for {}".
format(assets_dir, self))
shutil.rmtree(assets_dir)
os.makedirs(assets_dir)
logger.info(
"Downloading all.zip for {}".format(self))
# Download all assets
zip_stream = self.processing_node.download_task_asset(
self.uuid, "all.zip")
zip_path = os.path.join(assets_dir, "all.zip")
with open(zip_path, 'wb') as fd:
for chunk in zip_stream.iter_content(4096):
fd.write(chunk)
logger.info(
"Done downloading all.zip for {}".format(self))
# Extract from zip
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
logger.info(
"Extracted all.zip for {}".format(self))
# Populate *_extent fields
extent_fields = [
(os.path.realpath(
self.assets_path("odm_orthophoto",
"odm_orthophoto.tif")),
'orthophoto_extent'),
(os.path.realpath(
self.assets_path("odm_dem", "dsm.tif")),
'dsm_extent'),
(os.path.realpath(
self.assets_path("odm_dem", "dtm.tif")),
'dtm_extent'),
]
for raster_path, field in extent_fields:
if os.path.exists(raster_path):
# Read extent and SRID
raster = GDALRaster(raster_path)
extent = OGRGeometry.from_bbox(
raster.extent)
# It will be implicitly transformed into the SRID of the model’s field
# self.field = GEOSGeometry(...)
setattr(
self, field,
GEOSGeometry(extent.wkt,
srid=raster.srid))
logger.info(
"Populated extent field with {} for {}"
.format(raster_path, self))
self.update_available_assets_field()
self.running_progress = 1.0
self.save()
from app.plugins import signals as plugin_signals
plugin_signals.task_completed.send_robust(
sender=self.__class__, task_id=self.id)
else:
# FAILED, CANCELED
self.save()
else:
# Still waiting...
self.save()
except ProcessingError as e:
self.set_failure(str(e))
except (ConnectionRefusedError, ConnectionError) as e:
logger.warning(
"{} cannot communicate with processing node: {}".format(
self, str(e)))
except ProcessingTimeout as e:
logger.warning(
"{} timed out with error: {}. We'll try reprocessing at the next tick."
.format(self, str(e)))
print extent
bl = extent[:2]
tr = extent[2:]
tlt = deg2num(bl[0], tr[1], 14)
trt = deg2num(tr[0], tr[1], 14)
tlb = deg2num(bl[0], bl[1], 14)
xs = range(tlt[0], trt[0]+1)
ys = range(tlt[1], tlb[1]+1)
matrix = []
for xt in xs:
for yt in ys:
matrix.append((xt, yt))
for foo in matrix:
foo_1 = foo[1]
bbox = (foo[0] * 0.02197265625 - 180, num2deg(foo[1] + 1), (foo[0] + 1) * 0.02197265625 - 180, num2deg(foo[1]))
print bbox
dens = OGRGeometry.from_bbox(bbox)
dens.srid = 4326
d = TaskCell()
d.x = foo[0]
d.y = foo[1]
d.geom = dens.wkt
d.task = t
d.save()
#bl = sys.argv[1].split(",")[:2]
#ur = sys.argv[1].split(",")[2:]
#print bl
#print ur
blah = "-72.6775034,19.6069557,-72.5595274,19.6865932"
bblah = [ float(i) for i in blah.split(",")]
Density.objects.all().delete()
foo = OGRGeometry.from_bbox(bblah)
#print foo
#foo.srid = 900913
#foo.transform(4326)
print foo
#import ipdb; ipdb.set_trace()
# from this figure out what would be the polygons in tile size 15
# from this a grid model can be created and can keep track of the amount of nodes per tile
# zoom level 0 is a 1x1 tile (256)
# zoom level 1 will have 4 tiles
# zoom level 2 will have 16 tles
def test_burn_value_option(self):
geom = OGRGeometry.from_bbox((500000.0, 399800.0, 500200.0, 399900.0))
geom.srid = 3086
result = rasterize(geom, self.rast, burn_value=99)
self.assertEqual(result.bands[0].data().ravel().tolist(), [0, 0, 99, 99])
