class Criterion(models.Model):
name = models.CharField(max_length=150)
description = models.TextField(null=True, blank=True)
raster10 = models.RasterField(null=True, blank=True)
raster50 = models.RasterField(null=True, blank=True)
def __str__(self):
return self.name
class Result(models.Model):
user = models.OneToOneField(User,
on_delete=models.CASCADE,
primary_key=True)
commune = models.ManyToManyField(Commune, through='ResultCommune')
cr1_result_raster = models.RasterField(null=True, blank=True)
cr2_result_raster = models.RasterField(null=True, blank=True)
cr3_result_raster = models.RasterField(null=True, blank=True)
result_raster = models.RasterField(null=True, blank=True)
date_time = models.DateTimeField(auto_now=True)
class RasterModel(models.Model):
rast = models.RasterField('A Verbose Raster Name', null=True, srid=4326, spatial_index=True, blank=True)
rastprojected = models.RasterField('A Projected Raster Table', srid=3086, null=True)
geom = models.PointField(null=True)
class Meta:
required_db_features = ['supports_raster']
def __str__(self):
return str(self.id)
class LidarStats(models.Model):
name = models.CharField(max_length=250, null=True, blank=True)
height_raster = models.RasterField()
pulse_count_raster = models.RasterField()
returns = models.IntegerField(null=True, blank=True)
z_range = models.FloatField(null=True, blank=True)
mean_z = models.FloatField(null=True, blank=True)
std_dev = models.FloatField(null=True, blank=True)
x_range = models.FloatField(null=True, blank=True)
y_range = models.FloatField(null=True, blank=True)
def __str__(self):
return self.name
def get_tipo(self, tipo):
"""
Obtener el tipo de dato geometrico
"""
if tipo == Atributos.PUNTO:
return models.PointField()
elif tipo == Atributos.POLIGONO:
return models.PolygonField()
elif tipo == Atributos.POLIGONO_MULTIPLE:
return models.MultiPolygonField()
elif tipo == Atributos.PUNTO_MULTIPLE:
return models.MultiPointField()
elif tipo == Atributos.LINEA:
return models.LineStringField()
elif tipo == Atributos.LINEA_MULTIPLE:
return models.MultiLineStringField()
elif tipo == Atributos.TEXTO:
return models.CharField(max_length=255)
elif tipo == Atributos.ENTERO:
return models.IntegerField()
elif tipo == Atributos.FLOTANTE:
return models.FloatField()
elif tipo == Parametro.IMAGEN:
return models.ImageField(upload_to="/")
elif tipo == Parametro.DATE:
return models.DateField()
elif tipo == Parametro.DATETIME:
return models.DateTimeField()
else:
return models.RasterField()
class ImagemExemploTile2(TiffModel):
rid = models.AutoField(primary_key=True)
rast = models.RasterField(blank=True,
null=True) # This field type is a guess.
class Meta:
managed = False
db_table = 'imagem2_tiles'
class RasValue(models.Model):
""" """
prop = models.ForeignKey(Prop,
on_delete=models.CASCADE,
related_name='ras_values')
value = models.RasterField(srid=3857, null=True)
def __str__(self):
return 'raster, %s' % (self.id)
class Pizzeria(geo_models.Model):
hq = geo_models.PointField()
directions = geo_models.LineStringField()
floor_plan = geo_models.PolygonField()
locations = geo_models.MultiPointField()
routes = geo_models.MultiLineStringField()
delivery_areas = geo_models.MultiPolygonField()
all_the_things = geo_models.GeometryCollectionField()
rast = geo_models.RasterField()
class DEM(models.Model):
rast = models.RasterField()
class Meta:
db_table = 'dem'
objects = models.Manager()
profile = ProfileManager()
class DEM(models.Model):
rast = models.RasterField()
filename = models.CharField(max_length=50, blank=True, null=True)
class Meta:
db_table = 'dem'
objects = models.Manager()
profile = ProfileManager()
class RasterSeries(models.Model):
""" """
prop = models.ForeignKey(Prop,
on_delete=models.CASCADE,
related_name='raster_series')
value = models.RasterField(srid=3857)
timestamps = ArrayField(models.DateTimeField(null=True), null=True)
def __str__(self):
return '%s %s' % (self.id)
class Migration(migrations.Migration):
dependencies = []
operations = [
migrations.CreateModel(
name='RasterLayer',
fields=[
('id',
models.AutoField(verbose_name='ID',
serialize=False,
auto_created=True,
primary_key=True)),
('name', models.CharField(max_length=100,
null=True,
blank=True)),
('description', models.TextField(null=True, blank=True)),
('datatype',
models.CharField(default=b'co',
max_length=2,
choices=[(b'co', b'Continuous'),
(b'ca', b'Categorical'),
(b'ma', b'Mask'),
(b'ro', b'Rank Ordered')])),
('rasterfile', models.FileField(upload_to=b'rasters')),
('srid', models.CharField(default=b'3086', max_length=10)),
('nodata', models.CharField(default=b'-9999', max_length=100)),
('parse_log',
models.TextField(default=b'',
null=True,
editable=False,
blank=True)),
],
options={},
bases=(models.Model, ),
),
migrations.CreateModel(
name='RasterTile',
fields=[
('rid', models.AutoField(serialize=False, primary_key=True)),
('rast', models.RasterField(null=True, blank=True)),
('filename', models.TextField(null=True, blank=True)),
('rasterlayer',
models.ForeignKey(blank=True,
to='raster.RasterLayer',
null=True,
on_delete=models.CASCADE)),
],
options={},
bases=(models.Model, ),
),
]
class Feature(models.Model):
dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name="features")
raster = models.RasterField(null=True, blank=True)
geom_point = models.PointField(srid=4326, blank=True, null=True)
geom_multipoint = models.MultiPointField(srid=4326, blank=True, null=True)
geom_multilinestring = models.MultiLineStringField(srid=4326, blank=True, null=True)
geom_multipolygon = models.MultiPolygonField(srid=4326, blank=True, null=True)
geom_geometrycollection = models.GeometryCollectionField(srid=4326, blank=True, null=True)
attribute_values = JSONField(default=dict)
objects = GeoManager()
def __str__(self):
return str(self.id)
class Source(models.Model):
sid = models.CharField(max_length=15, verbose_name='Source Id.', primary_key=True)
mapdoc = models.RasterField(srid=32737, verbose_name='Map')
locality = models.CharField(max_length=40)
scale = models.CharField(max_length=15)
date_prep = models.DateField(default=timezone.now, verbose_name='Date Prepared')
surveyor = models.ForeignKey(Professional, on_delete=models.CASCADE)
class Meta:
abstract = True
ordering = ['sid']
def __str__(self):
return self.sid + ' - ' + self.locality
class RasterTile(models.Model):
class Meta:
verbose_name_plural = 'Raster Tiles'
rastertile_id = models.AutoField(primary_key=True)
rastertile_x = models.IntegerField(null=True, verbose_name='X')
rastertile_y = models.IntegerField(null=True, verbose_name='Y')
rastertile_zoom = models.IntegerField(null=True, verbose_name='Zoom')
rastertile_size = models.IntegerField(default=512,
verbose_name='Tile size')
rastertile_layer = models.ForeignKey(RasterLayer, on_delete=models.CASCADE)
rast = models.RasterField(srid=3857)
def to_png(self):
""" Convert raster to png """
raster = self.rast
gdal_bands = np.array(
[raster.bands[x].data() for x in range(len(raster.bands))])
nodata = raster.bands[0].nodata_value
rgb = np.rollaxis(gdal_bands, 0, 3)
a = ((np.sum(rgb, axis=2) != nodata * 3) * 255).astype(np.uint8)
p_a = Image.fromarray(a.astype(np.uint8), 'L')
p_rgb = Image.fromarray(rgb.astype(np.uint8))
p_rgb.putalpha(p_a)
buffer = BytesIO()
p_rgb.save(fp=buffer, format="PNG")
return buffer
def __str__(self):
params = (self.rastertile_layer.rasterlayer_name, self.rastertile_zoom,
self.rastertile_x, self.rastertile_y)
return '%s [Z=%d X=%d Y=%d]' % params
def __repr__(self):
return self.__str__()
@staticmethod
def get(rasterlayer, zoom, x, y):
return RasterTile.objects.filter(rastertile_layer=rasterlayer,
rastertile_x=x,
rastertile_y=y,
rastertile_zoom=zoom).first()
class RasterTile(models.Model):
"""
Store individual tiles of a raster data source layer.
"""
ZOOMLEVELS = (
(1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7),
(8, 8), (9, 9), (10, 10), (11, 11), (12, 12), (13, 13),
(14, 14), (15, 15), (16, 16), (17, 17), (18, 18)
)
rid = models.AutoField(primary_key=True)
rast = models.RasterField(null=True, blank=True, srid=WEB_MERCATOR_SRID)
rasterlayer = models.ForeignKey(RasterLayer, null=True, blank=True, db_index=True)
tilex = models.IntegerField(db_index=True, null=True)
tiley = models.IntegerField(db_index=True, null=True)
tilez = models.IntegerField(db_index=True, null=True, choices=ZOOMLEVELS)
def __str__(self):
return '{} {}'.format(self.rid, self.rasterlayer.name)
class Task(models.Model):
STATUS_CODES = (
(status_codes.QUEUED, 'QUEUED'),
(status_codes.RUNNING, 'RUNNING'),
(status_codes.FAILED, 'FAILED'),
(status_codes.COMPLETED, 'COMPLETED'),
(status_codes.CANCELED, 'CANCELED')
)
PENDING_ACTIONS = (
(pending_actions.CANCEL, 'CANCEL'),
(pending_actions.REMOVE, 'REMOVE'),
(pending_actions.RESTART, 'RESTART'),
)
uuid = models.CharField(max_length=255, db_index=True, default='', blank=True, help_text="Identifier of the task (as returned by OpenDroneMap's REST API)")
project = models.ForeignKey(Project, on_delete=models.CASCADE, help_text="Project that this task belongs to")
name = models.CharField(max_length=255, null=True, blank=True, help_text="A label for the task")
processing_lock = models.BooleanField(default=False, help_text="A flag indicating whether this task is currently locked for processing. When this flag is turned on, the task is in the middle of a processing step.")
processing_time = models.IntegerField(default=-1, help_text="Number of milliseconds that elapsed since the beginning of this task (-1 indicates that no information is available)")
processing_node = models.ForeignKey(ProcessingNode, null=True, blank=True, help_text="Processing node assigned to this task (or null if this task has not been associated yet)")
status = models.IntegerField(choices=STATUS_CODES, db_index=True, null=True, blank=True, help_text="Current status of the task")
last_error = models.TextField(null=True, blank=True, help_text="The last processing error received")
options = fields.JSONField(default=dict(), blank=True, help_text="Options that are being used to process this task", validators=[validate_task_options])
console_output = models.TextField(null=False, default="", blank=True, help_text="Console output of the OpenDroneMap's process")
ground_control_points = models.FileField(null=True, blank=True, upload_to=gcp_directory_path, help_text="Optional Ground Control Points file to use for processing")
# georeferenced_model
orthophoto = gismodels.RasterField(null=True, blank=True, srid=4326, help_text="Orthophoto created by OpenDroneMap")
# textured_model
# mission
created_at = models.DateTimeField(default=timezone.now, help_text="Creation date")
pending_action = models.IntegerField(choices=PENDING_ACTIONS, db_index=True, null=True, blank=True, help_text="A requested action to be performed on the task. The selected action will be performed by the scheduler at the next iteration.")
def __str__(self):
return 'Task ID: {}'.format(self.id)
def save(self, *args, **kwargs):
# Autovalidate on save
self.full_clean()
super(Task, self).save(*args, **kwargs)
def assets_path(self, *args):
"""
Get a path relative to the place where assets are stored
"""
return os.path.join(settings.MEDIA_ROOT,
assets_directory_path(self.id, self.project.id, ""),
"assets",
*args)
@staticmethod
def create_from_images(images, project):
'''
Create a new task from a set of input images (such as the ones coming from request.FILES).
This will happen inside a transaction so if one of the images
fails to load, the task will not be created.
'''
with transaction.atomic():
task = Task.objects.create(project=project)
for image in images:
ImageUpload.objects.create(task=task, image=image)
return task
def process(self):
"""
This method contains the logic for processing tasks asynchronously
from a background thread or from the scheduler. Here tasks that are
ready to be processed execute some logic. This could be communication
with a processing node or executing a pending action.
"""
if self.processing_node:
# Need to process some images (UUID not yet set)?
if not self.uuid:
logger.info("Processing... {}".format(self))
images = [image.path() for image in self.imageupload_set.all()]
try:
# This takes a while
uuid = self.processing_node.process_new_task(images, self.name, self.options)
# Refresh task object before committing change
self.refresh_from_db()
self.uuid = uuid
self.save()
# TODO: log process has started processing
except ProcessingException as e:
self.set_failure(str(e))
if self.pending_action is not None:
try:
if self.pending_action == pending_actions.CANCEL:
# Do we need to cancel the task on the processing node?
logger.info("Canceling task {}".format(self))
if self.processing_node and self.uuid:
self.processing_node.cancel_task(self.uuid)
self.pending_action = None
self.save()
else:
raise ProcessingException("Cannot cancel a task that has no processing node or UUID")
elif self.pending_action == pending_actions.RESTART:
logger.info("Restarting task {}".format(self))
if self.processing_node and self.uuid:
# Check if the UUID is still valid, as processing nodes purge
# results after a set amount of time, the UUID might have eliminated.
try:
info = self.processing_node.get_task_info(self.uuid)
uuid_still_exists = info['uuid'] == self.uuid
except ProcessingException:
uuid_still_exists = False
if uuid_still_exists:
# Good to go
self.processing_node.restart_task(self.uuid)
else:
# Task has been purged (or processing node is offline)
# TODO: what if processing node went offline?
# Process this as a new task
# Removing its UUID will cause the scheduler
# to process this the next tick
self.uuid = None
self.console_output = ""
self.processing_time = -1
self.status = None
self.last_error = None
self.pending_action = None
self.save()
else:
raise ProcessingException("Cannot restart a task that has no processing node or UUID")
elif self.pending_action == pending_actions.REMOVE:
logger.info("Removing task {}".format(self))
if self.processing_node and self.uuid:
# Attempt to delete the resources on the processing node
# We don't care if this fails, as resources on processing nodes
# Are expected to be purged on their own after a set amount of time anyway
try:
self.processing_node.remove_task(self.uuid)
except ProcessingException:
pass
# What's more important is that we delete our task properly here
self.delete()
# Stop right here!
return
except ProcessingException as e:
self.last_error = str(e)
self.save()
if self.processing_node:
# Need to update status (first time, queued or running?)
if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]:
# Update task info from processing node
try:
info = self.processing_node.get_task_info(self.uuid)
self.processing_time = info["processingTime"]
self.status = info["status"]["code"]
current_lines_count = len(self.console_output.split("\n")) - 1
self.console_output += self.processing_node.get_task_console_output(self.uuid, current_lines_count)
if "errorMessage" in info["status"]:
self.last_error = info["status"]["errorMessage"]
# Has the task just been canceled, failed, or completed?
if self.status in [status_codes.FAILED, status_codes.COMPLETED, status_codes.CANCELED]:
logger.info("Processing status: {} for {}".format(self.status, self))
if self.status == status_codes.COMPLETED:
try:
assets_dir = self.assets_path("")
if not os.path.exists(assets_dir):
os.makedirs(assets_dir)
# Download all assets
zip_stream = self.processing_node.download_task_asset(self.uuid, "all.zip")
zip_path = os.path.join(assets_dir, "all.zip")
with open(zip_path, 'wb') as fd:
for chunk in zip_stream.iter_content(4096):
fd.write(chunk)
# Extract from zip
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
# Add to database orthophoto
orthophoto_path = self.assets_path("odm_orthophoto", "odm_orthophoto.tif")
if os.path.exists(orthophoto_path):
self.orthophoto = GDALRaster(orthophoto_path, write=True)
self.save()
except ProcessingException as e:
self.set_failure(str(e))
else:
# FAILED, CANCELED
self.save()
else:
# Still waiting...
self.save()
except ProcessingException as e:
self.set_failure(str(e))
def get_tile_path(self, z, x, y):
return self.assets_path("orthophoto_tiles", z, x, "{}.png".format(y))
def get_tile_json_url(self):
return "/api/projects/{}/tasks/{}/tiles.json".format(self.project.id, self.id)
def get_tile_json_data(self):
return {
'url': self.get_tile_json_url(),
'meta': {
'task': self.id,
'project': self.project.id
}
}
def delete(self, using=None, keep_parents=False):
directory_to_delete = os.path.join(settings.MEDIA_ROOT,
task_directory_path(self.id, self.project.id))
super(Task, self).delete(using, keep_parents)
# Remove files related to this task
try:
shutil.rmtree(directory_to_delete)
except FileNotFoundError as e:
logger.warn(e)
def set_failure(self, error_message):
logger.error("{} ERROR: {}".format(self, error_message))
self.last_error = error_message
self.status = status_codes.FAILED
self.save()
class Meta:
permissions = (
('view_task', 'Can view task'),
)
class Dem(models.Model):
id = models.AutoField(
primary_key=True,
db_column='rid') # rid is id column name used by raster2pgsql
rast = models.RasterField(srid=settings.SRID)
class Elevation(BaseModel):
name = models.CharField(max_length=100)
rast = models.RasterField()
class Elevation(models.Model):
name = models.CharField(max_length=100)
rast = models.RasterField(blank=True, null=True)
class RasterFieldModel(models.Model):
rast = models.RasterField()
def __str__(self):
return str(self.id)
class CoverageRaster(models.Model):
raster = models.ForeignKey(Raster, on_delete=models.CASCADE)
cov_rast = models.RasterField(srid=32718)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
