def as_postgresql(self, compiler, connection):
function = None
geo_field = GeometryField(
srid=self.srid) # Fake field to get SRID info
expr2 = self.source_expressions[1]
geography = self.source_is_geography()
if expr2.output_field.geography != geography:
if isinstance(expr2, Value):
expr2.output_field.geography = geography
else:
self.source_expressions[1] = Cast(
expr2,
GeometryField(srid=expr2.output_field.srid,
geography=geography),
)
if not geography and geo_field.geodetic(connection):
# Geometry fields with geodetic (lon/lat) coordinates need special distance functions
if self.spheroid:
# DistanceSpheroid is more accurate and resource intensive than DistanceSphere
function = connection.ops.spatial_function_name(
'DistanceSpheroid')
# Replace boolean param by the real spheroid of the base field
self.source_expressions[2] = Value(
geo_field.spheroid(connection))
else:
function = connection.ops.spatial_function_name(
'DistanceSphere')
return super().as_sql(compiler, connection, function=function)
def as_sqlite(self, compiler, connection):
geo_field = GeometryField(
srid=self.srid) # Fake field to get SRID info
if geo_field.geodetic(connection):
raise NotImplementedError(
"Perimeter cannot use a non-projected field.")
return super(Perimeter, self).as_sql(compiler, connection)
def as_sqlite(self, compiler, connection):
geo_field = GeometryField(
srid=self.srid) # Fake field to get SRID info
if geo_field.geodetic(connection):
raise NotImplementedError(
"Perimeter cannot use a non-projected field.")
return super(Perimeter, self).as_sql(compiler, connection)
def as_postgresql(self, compiler, connection):
geo_field = GeometryField(srid=self.srid) # Fake field to get SRID info
if geo_field.geodetic(connection) and not self.source_is_geography():
raise NotImplementedError("ST_Perimeter cannot use a non-projected non-geography field.")
dim = min(f.dim for f in self.get_source_fields())
if dim > 2:
self.function = connection.ops.perimeter3d
return super(Perimeter, self).as_sql(compiler, connection)
def as_sqlite(self, compiler, connection):
geo_field = GeometryField(srid=self.srid)
if geo_field.geodetic(connection):
if self.spheroid:
self.function = 'GeodesicLength'
else:
self.function = 'GreatCircleLength'
return super(Length, self).as_sql(compiler, connection)
def as_sqlite(self, compiler, connection):
geo_field = GeometryField(srid=self.srid)
if geo_field.geodetic(connection):
if self.spheroid:
self.function = 'GeodesicLength'
else:
self.function = 'GreatCircleLength'
return super(Length, self).as_sql(compiler, connection)
def as_sql(self, compiler, connection):
geo_field = GeometryField(
srid=self.srid) # Fake field to get SRID info
if geo_field.geodetic(
connection
) and not connection.features.supports_length_geodetic:
raise NotImplementedError(
"This backend doesn't support Length on geodetic fields")
return super(Length, self).as_sql(compiler, connection)
def as_sql(self, compiler, connection):
geo_field = GeometryField(
srid=self.srid) # Fake field to get SRID info
if geo_field.geodetic(
connection
) and not connection.features.supports_length_geodetic:
raise NotImplementedError(
"This backend doesn't support Length on geodetic fields")
return super(Length, self).as_sql(compiler, connection)
def as_postgresql(self, compiler, connection):
geo_field = GeometryField(srid=self.srid) # Fake field to get SRID info
if self.source_is_geography():
self.source_expressions.append(Value(self.spheroid))
elif geo_field.geodetic(connection):
# Geometry fields with geodetic (lon/lat) coordinates need length_spheroid
self.function = 'ST_Length_Spheroid'
self.source_expressions.append(Value(geo_field._spheroid))
else:
dim = min(f.dim for f in self.get_source_fields() if f)
if dim > 2:
self.function = connection.ops.length3d
return super(Length, self).as_sql(compiler, connection)
def as_postgresql(self, compiler, connection):
function = None
geo_field = GeometryField(srid=self.srid) # Fake field to get SRID info
if self.source_is_geography():
self.source_expressions.append(Value(self.spheroid))
elif geo_field.geodetic(connection):
# Geometry fields with geodetic (lon/lat) coordinates need length_spheroid
function = connection.ops.spatial_function_name('LengthSpheroid')
self.source_expressions.append(Value(geo_field.spheroid(connection)))
else:
dim = min(f.dim for f in self.get_source_fields() if f)
if dim > 2:
function = connection.ops.length3d
return super().as_sql(compiler, connection, function=function)
def convert_value(self, value, expression, connection, context):
if value is None:
return None
geo_field = GeometryField(srid=self.srid) # Fake field to get SRID info
if geo_field.geodetic(connection):
dist_att = 'm'
else:
units = geo_field.units_name(connection)
if units:
dist_att = DistanceMeasure.unit_attname(units)
else:
dist_att = None
if dist_att:
return DistanceMeasure(**{dist_att: value})
return value
def convert_value(self, value, expression, connection, context):
if value is None:
return None
geo_field = GeometryField(srid=self.srid) # Fake field to get SRID info
if geo_field.geodetic(connection):
dist_att = 'm'
else:
units = geo_field.units_name(connection)
if units:
dist_att = DistanceMeasure.unit_attname(units)
else:
dist_att = None
if dist_att:
return DistanceMeasure(**{dist_att: value})
return value
def as_postgresql(self, compiler, connection):
clone = self.copy()
function = None
expr2 = clone.source_expressions[1]
geography = self.source_is_geography()
if expr2.output_field.geography != geography:
if isinstance(expr2, Value):
expr2.output_field.geography = geography
else:
clone.source_expressions[1] = Cast(
expr2,
GeometryField(srid=expr2.output_field.srid,
geography=geography),
)
if not geography and self.geo_field.geodetic(connection):
# Geometry fields with geodetic (lon/lat) coordinates need special distance functions
if self.spheroid:
# DistanceSpheroid is more accurate and resource intensive than DistanceSphere
function = connection.ops.spatial_function_name(
"DistanceSpheroid")
# Replace boolean param by the real spheroid of the base field
clone.source_expressions.append(
Value(self.geo_field.spheroid(connection)))
else:
function = connection.ops.spatial_function_name(
"DistanceSphere")
return super(Distance, clone).as_sql(compiler,
connection,
function=function)
def __init__(self, *expressions, **extra):
if 'output_field' not in extra and self.output_field_class:
extra['output_field'] = self.output_field_class()
super().__init__(*expressions, **extra)
# Ensure that value expressions are geometric.
for pos in self.geom_param_pos:
expr = self.source_expressions[pos]
if not isinstance(expr, Value):
continue
try:
output_field = expr.output_field
except FieldError:
output_field = None
geom = expr.value
if not isinstance(geom,
Geometry) or output_field and not isinstance(
output_field, GeometryField):
raise TypeError(
"%s function requires a geometric argument in position %d."
% (self.name, pos + 1))
if not geom.srid and not output_field:
raise ValueError("SRID is required for all geometries.")
if not output_field:
self.source_expressions[pos] = Value(
geom, output_field=GeometryField(srid=geom.srid))
def test_deconstruct_values(self):
field = GeometryField(
srid=4067,
dim=3,
geography=True,
extent=(50199.4814, 6582464.0358, -50000.0, 761274.6247, 7799839.8902, 50000.0),
tolerance=0.01,
)
*_, kwargs = field.deconstruct()
self.assertEqual(kwargs, {
'srid': 4067,
'dim': 3,
'geography': True,
'extent': (50199.4814, 6582464.0358, -50000.0, 761274.6247, 7799839.8902, 50000.0),
'tolerance': 0.01,
})
class ZipCode(models.Model):
code = models.CharField(max_length=5, unique=True)
bounds = GeometryField(geography=True)
center = PointField()
def __str__(self):
return self.code
class Location(models.Model):
"""
Provide a location, a geospatial polygon describing a part of the school::
>>> library = Location.objects.create(name='Library',
... polygon='POLYGON ((1 1, 3 3, 3 1, 1 1))')
>>> library.save()
>>> print(library)
Library
>>> print(library.polygon)
SRID=4326;POLYGON ((1 1, 3 3, 3 1, 1 1))
Locate the centre of the polygon using ``STCentroid()``.
>>> print(library.polygon.centroid)
SRID=4326;POINT (2.333333333333334 1.666666666666667)
"""
locationid = models.AutoField(primary_key=True)
name = models.CharField(
max_length=50,
help_text=_(
"Useful name for this location - by convention uses CamelCase"))
polygon = GeometryField(help_text=_("Polygon or Multipolygon"))
def __str__(self):
"Returns the location's name."
return self.name
class Meta:
ordering = ('name', )
def as_postgresql(self, compiler, connection):
function = None
geo_field = GeometryField(
srid=self.srid) # Fake field to get SRID info
if self.source_is_geography():
self.source_expressions.append(Value(self.spheroid))
elif geo_field.geodetic(connection):
# Geometry fields with geodetic (lon/lat) coordinates need length_spheroid
function = connection.ops.spatial_function_name('LengthSpheroid')
self.source_expressions.append(
Value(geo_field.spheroid(connection)))
else:
dim = min(f.dim for f in self.get_source_fields() if f)
if dim > 2:
function = connection.ops.length3d
return super().as_sql(compiler, connection, function=function)
def __init__(self, expression, srid, **extra):
expressions = [
expression,
self._handle_param(srid, 'srid', int),
]
if 'output_field' not in extra:
extra['output_field'] = GeometryField(srid=srid)
super().__init__(*expressions, **extra)
def __init__(self, expression, srid, **extra):
expressions = [
expression,
self._handle_param(srid, "srid", int),
]
if "output_field" not in extra:
extra["output_field"] = GeometryField(srid=srid)
super().__init__(*expressions, **extra)
def as_postgresql(self, compiler, connection):
geo_field = GeometryField(srid=self.srid) # Fake field to get SRID info
if self.source_is_geography():
# Set parameters as geography if base field is geography
for pos, expr in enumerate(
self.source_expressions[self.geom_param_pos + 1:], start=self.geom_param_pos + 1):
if isinstance(expr, GeomValue):
expr.geography = True
elif geo_field.geodetic(connection):
# Geometry fields with geodetic (lon/lat) coordinates need special distance functions
if self.spheroid:
self.function = 'ST_Distance_Spheroid' # More accurate, resource intensive
# Replace boolean param by the real spheroid of the base field
self.source_expressions[2] = Value(geo_field._spheroid)
else:
self.function = 'ST_Distance_Sphere'
return super(Distance, self).as_sql(compiler, connection)
def __init__(self, expression, srid, **extra):
expressions = [
expression,
self._handle_param(srid, 'srid', six.integer_types),
]
if 'output_field' not in extra:
extra['output_field'] = GeometryField(srid=srid)
super(Transform, self).__init__(*expressions, **extra)
class vols(models.Model):
#id = models.UUIDField(primary_key=True, default=uuid_module.uuid4, unique=True, serialize=False, editable=False)
created = models.DateTimeField(auto_now_add=True)
name = models.CharField(max_length=150)
id_vol = models.IntegerField(default=0)
donnees = models.CharField(max_length=1500)
#ortho = models.RasterField()
orthophoto = GeometryField(
null=True,
blank=True,
srid=4326,
help_text="Extent of the orthophoto created by OpenDroneMap")
dsm_extent = GeometryField(
null=True,
blank=True,
srid=4326,
help_text="Extent of the DSM created by OpenDroneMap")
dtm_extent = GeometryField(
null=True,
blank=True,
srid=4326,
help_text="Extent of the DTM created by OpenDroneMap")
def test_deconstruct_values(self):
field = GeometryField(
srid=4067,
dim=3,
geography=True,
extent=(50199.4814, 6582464.0358, -50000.0, 761274.6247,
7799839.8902, 50000.0),
tolerance=0.01,
)
*_, kwargs = field.deconstruct()
self.assertEqual(
kwargs, {
'srid':
4067,
'dim':
3,
'geography':
True,
'extent': (50199.4814, 6582464.0358, -50000.0, 761274.6247,
7799839.8902, 50000.0),
'tolerance':
0.01,
})
def as_postgresql(self, compiler, connection):
function = None
geo_field = GeometryField(
srid=self.srid) # Fake field to get SRID info
if self.source_is_geography():
# Set parameters as geography if base field is geography
for pos, expr in enumerate(
self.source_expressions[self.geom_param_pos + 1:],
start=self.geom_param_pos + 1):
if isinstance(expr, GeomValue):
expr.geography = True
elif geo_field.geodetic(connection):
# Geometry fields with geodetic (lon/lat) coordinates need special distance functions
if self.spheroid:
# DistanceSpheroid is more accurate and resource intensive than DistanceSphere
function = connection.ops.spatial_function_name(
'DistanceSpheroid')
# Replace boolean param by the real spheroid of the base field
self.source_expressions[2] = Value(geo_field._spheroid)
else:
function = connection.ops.spatial_function_name(
'DistanceSphere')
return super().as_sql(compiler, connection, function=function)
def as_postgresql(self, compiler, connection):
function = None
geo_field = GeometryField(srid=self.srid) # Fake field to get SRID info
expr2 = self.source_expressions[1]
geography = self.source_is_geography()
if expr2.output_field.geography != geography:
if isinstance(expr2, Value):
expr2.output_field.geography = geography
else:
self.source_expressions[1] = Cast(
expr2,
GeometryField(srid=expr2.output_field.srid, geography=geography),
)
if not geography and geo_field.geodetic(connection):
# Geometry fields with geodetic (lon/lat) coordinates need special distance functions
if self.spheroid:
# DistanceSpheroid is more accurate and resource intensive than DistanceSphere
function = connection.ops.spatial_function_name('DistanceSpheroid')
# Replace boolean param by the real spheroid of the base field
self.source_expressions[2] = Value(geo_field.spheroid(connection))
else:
function = connection.ops.spatial_function_name('DistanceSphere')
return super().as_sql(compiler, connection, function=function)
def as_sqlite(self, compiler, connection):
function = None
geo_field = GeometryField(srid=self.srid)
if geo_field.geodetic(connection):
function = 'GeodesicLength' if self.spheroid else 'GreatCircleLength'
return super().as_sql(compiler, connection, function=function)
def geo_field(self):
return GeometryField(srid=self.srid) if self.srid else None
class Location(models.Model):
id = models.BigIntegerField(primary_key=True)
name = models.TextField(blank=True, null=True)
division_id = models.TextField(blank=True, null=True)
feature_type = models.TextField(blank=True, null=True)
tags = models.JSONField(blank=True, null=True)
sfm = models.JSONField(blank=True, null=True)
adminlevel1 = models.ForeignKey('self',
related_name='area_locations',
on_delete=models.CASCADE,
null=True,
blank=True)
adminlevel2 = models.ForeignKey('self',
related_name='place_locations',
on_delete=models.CASCADE,
null=True,
blank=True)
adminlevel = models.CharField(max_length=50, null=True, blank=True)
geometry = GeometryField(blank=True, null=True)
objects = LocationManager()
def __str__(self):
if self.name is None:
return str(self.id)
return self.name
@property
def related_entities(self):
"""
Return a list of dicts with metadata for all of the entities (Organizations
and Violations) linked to this Location via an Emplacement or Association.
Metadata dicts must have the following keys:
- name
- entity_type
- start_date
- end_date
- open_ended
- url (a link to edit the entity)
"""
related_entities = []
for associationarea in self.associationarea_set.all():
association = associationarea.object_ref
organization = association.organization.get_value().value
related_entities.append({
'name':
organization.name.get_value().value,
'entity_type':
_('Organization'),
'start_date':
association.startdate.get_value(),
'end_date':
association.enddate.get_value(),
'open_ended':
association.open_ended.get_value(),
'url':
reverse('view-organization',
kwargs={'slug': organization.uuid}),
})
for emplacementsite in self.emplacementsite_set.all():
emplacement = emplacementsite.object_ref
organization = emplacement.organization.get_value().value
related_entities.append({
'name':
organization.name.get_value().value,
'entity_type':
_('Organization'),
'start_date':
emplacement.startdate.get_value(),
'end_date':
emplacement.enddate.get_value(),
'open_ended':
emplacement.open_ended.get_value(),
'url':
reverse('view-organization',
kwargs={'slug': organization.uuid}),
})
for violationlocation in self.violationlocation_set.all():
violation = violationlocation.object_ref
related_entities.append({
'name':
truncatewords(violation.description.get_value(), 10),
'entity_type':
_('Violation'),
'start_date':
violation.startdate.get_value(),
'end_date':
violation.enddate.get_value(),
'open_ended':
'',
'url':
reverse('view-violation', kwargs={'slug': violation.uuid}),
})
return related_entities
@property
def osm_feature_type(self):
if self.feature_type == 'boundary':
return 'relation'
return self.feature_type
class Task(models.Model):
ASSETS_MAP = {
'all.zip':
'all.zip',
'orthophoto.tif':
os.path.join('odm_orthophoto', 'odm_orthophoto.tif'),
'orthophoto.png':
os.path.join('odm_orthophoto', 'odm_orthophoto.png'),
'orthophoto.mbtiles':
os.path.join('odm_orthophoto', 'odm_orthophoto.mbtiles'),
'georeferenced_model.las':
os.path.join('odm_georeferencing', 'odm_georeferenced_model.las'),
'georeferenced_model.laz':
os.path.join('odm_georeferencing', 'odm_georeferenced_model.laz'),
'georeferenced_model.ply':
os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply'),
'georeferenced_model.csv':
os.path.join('odm_georeferencing', 'odm_georeferenced_model.csv'),
'textured_model.zip': {
'deferred_path': 'textured_model.zip',
'deferred_compress_dir': 'odm_texturing'
},
'dtm.tif':
os.path.join('odm_dem', 'dtm.tif'),
'dsm.tif':
os.path.join('odm_dem', 'dsm.tif'),
}
STATUS_CODES = (
(status_codes.QUEUED, 'QUEUED'),
(status_codes.RUNNING, 'RUNNING'),
(status_codes.FAILED, 'FAILED'),
(status_codes.COMPLETED, 'COMPLETED'),
(status_codes.CANCELED, 'CANCELED'),
)
PENDING_ACTIONS = (
(pending_actions.CANCEL, 'CANCEL'),
(pending_actions.REMOVE, 'REMOVE'),
(pending_actions.RESTART, 'RESTART'),
(pending_actions.RESIZE, 'RESIZE'),
)
# Not an exact science
TASK_OUTPUT_MILESTONES = {
'Running ODM Load Dataset Cell': 0.01,
'Running ODM Load Dataset Cell - Finished': 0.05,
'opensfm/bin/opensfm match_features': 0.10,
'opensfm/bin/opensfm reconstruct': 0.20,
'opensfm/bin/opensfm export_visualsfm': 0.30,
'Running ODM Meshing Cell': 0.60,
'Running MVS Texturing Cell': 0.65,
'Running ODM Georeferencing Cell': 0.70,
'Running ODM DEM Cell': 0.80,
'Running ODM Orthophoto Cell': 0.85,
'Running ODM OrthoPhoto Cell - Finished': 0.90,
'Compressing all.zip:': 0.95
}
id = models.UUIDField(primary_key=True,
default=uuid_module.uuid4,
unique=True,
serialize=False,
editable=False)
uuid = models.CharField(
max_length=255,
db_index=True,
default='',
blank=True,
help_text=
"Identifier of the task (as returned by OpenDroneMap's REST API)")
project = models.ForeignKey(Project,
on_delete=models.CASCADE,
help_text="Project that this task belongs to")
name = models.CharField(max_length=255,
null=True,
blank=True,
help_text="A label for the task")
processing_time = models.IntegerField(
default=-1,
help_text=
"Number of milliseconds that elapsed since the beginning of this task (-1 indicates that no information is available)"
)
processing_node = models.ForeignKey(
ProcessingNode,
on_delete=models.SET_NULL,
null=True,
blank=True,
help_text=
"Processing node assigned to this task (or null if this task has not been associated yet)"
)
auto_processing_node = models.BooleanField(
default=True,
help_text=
"A flag indicating whether this task should be automatically assigned a processing node"
)
status = models.IntegerField(choices=STATUS_CODES,
db_index=True,
null=True,
blank=True,
help_text="Current status of the task")
last_error = models.TextField(
null=True, blank=True, help_text="The last processing error received")
options = fields.JSONField(
default=dict(),
blank=True,
help_text="Options that are being used to process this task",
validators=[validate_task_options])
available_assets = fields.ArrayField(
models.CharField(max_length=80),
default=list(),
blank=True,
help_text="List of available assets to download")
console_output = models.TextField(
null=False,
default="",
blank=True,
help_text="Console output of the OpenDroneMap's process")
orthophoto_extent = GeometryField(
null=True,
blank=True,
srid=4326,
help_text="Extent of the orthophoto created by OpenDroneMap")
dsm_extent = GeometryField(
null=True,
blank=True,
srid=4326,
help_text="Extent of the DSM created by OpenDroneMap")
dtm_extent = GeometryField(
null=True,
blank=True,
srid=4326,
help_text="Extent of the DTM created by OpenDroneMap")
# mission
created_at = models.DateTimeField(default=timezone.now,
help_text="Creation date")
pending_action = models.IntegerField(
choices=PENDING_ACTIONS,
db_index=True,
null=True,
blank=True,
help_text=
"A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration."
)
public = models.BooleanField(
default=False,
help_text=
"A flag indicating whether this task is available to the public")
resize_to = models.IntegerField(
default=-1,
help_text=
"When set to a value different than -1, indicates that the images for this task have been / will be resized to the size specified here before processing."
)
upload_progress = models.FloatField(
default=0.0,
help_text=
"Value between 0 and 1 indicating the upload progress of this task's files to the processing node",
blank=True)
resize_progress = models.FloatField(
default=0.0,
help_text=
"Value between 0 and 1 indicating the resize progress of this task's images",
blank=True)
running_progress = models.FloatField(
default=0.0,
help_text=
"Value between 0 and 1 indicating the running progress (estimated) of this task",
blank=True)
def __init__(self, *args, **kwargs):
super(Task, self).__init__(*args, **kwargs)
# To help keep track of changes to the project id
self.__original_project_id = self.project.id
def __str__(self):
name = self.name if self.name is not None else "unnamed"
return 'Task [{}] ({})'.format(name, self.id)
def move_assets(self, old_project_id, new_project_id):
"""
Moves the task's folder, update ImageFields and orthophoto files to a new project
"""
old_task_folder = full_task_directory_path(self.id, old_project_id)
new_task_folder = full_task_directory_path(self.id, new_project_id)
new_task_folder_parent = os.path.abspath(
os.path.join(new_task_folder, os.pardir))
try:
if os.path.exists(
old_task_folder) and not os.path.exists(new_task_folder):
# Use parent, otherwise we get a duplicate directory in there
if not os.path.exists(new_task_folder_parent):
os.makedirs(new_task_folder_parent)
shutil.move(old_task_folder, new_task_folder_parent)
logger.info("Moved task folder from {} to {}".format(
old_task_folder, new_task_folder))
with transaction.atomic():
for img in self.imageupload_set.all():
prev_name = img.image.name
img.image.name = assets_directory_path(
self.id, new_project_id,
os.path.basename(img.image.name))
logger.info("Changing {} to {}".format(prev_name, img))
img.save()
else:
logger.warning(
"Project changed for task {}, but either {} doesn't exist, or {} already exists. This doesn't look right, so we will not move any files."
.format(self, old_task_folder, new_task_folder))
except shutil.Error as e:
logger.warning(
"Could not move assets folder for task {}. We're going to proceed anyway, but you might experience issues: {}"
.format(self, e))
def save(self, *args, **kwargs):
if self.project.id != self.__original_project_id:
self.move_assets(self.__original_project_id, self.project.id)
self.__original_project_id = self.project.id
# Autovalidate on save
self.full_clean()
super(Task, self).save(*args, **kwargs)
def assets_path(self, *args):
"""
Get a path relative to the place where assets are stored
"""
return self.task_path("assets", *args)
def task_path(self, *args):
"""
Get path relative to the root task directory
"""
return os.path.join(
settings.MEDIA_ROOT,
assets_directory_path(self.id, self.project.id, ""), *args)
def is_asset_available_slow(self, asset):
"""
Checks whether a particular asset is available in the file system
Generally this should never be used directly, as it's slow. Use the available_assets field
in the database instead.
:param asset: one of ASSETS_MAP keys
:return: boolean
"""
if asset in self.ASSETS_MAP:
value = self.ASSETS_MAP[asset]
if isinstance(value, str):
return os.path.exists(self.assets_path(value))
elif isinstance(value, dict):
if 'deferred_compress_dir' in value:
return os.path.exists(
self.assets_path(value['deferred_compress_dir']))
return False
def get_asset_download_path(self, asset):
"""
Get the path to an asset download
:param asset: one of ASSETS_MAP keys
:return: path
"""
if asset in self.ASSETS_MAP:
value = self.ASSETS_MAP[asset]
if isinstance(value, str):
return self.assets_path(value)
elif isinstance(value, dict):
if 'deferred_path' in value and 'deferred_compress_dir' in value:
return self.generate_deferred_asset(
value['deferred_path'], value['deferred_compress_dir'])
else:
raise FileNotFoundError(
"{} is not a valid asset (invalid dict values)".format(
asset))
else:
raise FileNotFoundError(
"{} is not a valid asset (invalid map)".format(asset))
else:
raise FileNotFoundError("{} is not a valid asset".format(asset))
def process(self):
"""
This method contains the logic for processing tasks asynchronously
from a background thread or from a worker. Here tasks that are
ready to be processed execute some logic. This could be communication
with a processing node or executing a pending action.
"""
try:
if self.pending_action == pending_actions.RESIZE:
resized_images = self.resize_images()
self.refresh_from_db()
self.resize_gcp(resized_images)
self.pending_action = None
self.save()
if self.auto_processing_node and not self.status in [
status_codes.FAILED, status_codes.CANCELED
]:
# No processing node assigned and need to auto assign
if self.processing_node is None:
# Assign first online node with lowest queue count
self.processing_node = ProcessingNode.find_best_available_node(
)
if self.processing_node:
self.processing_node.queue_count += 1 # Doesn't have to be accurate, it will get overridden later
self.processing_node.save()
logger.info(
"Automatically assigned processing node {} to {}".
format(self.processing_node, self))
self.save()
# Processing node assigned, but is offline and no errors
if self.processing_node and not self.processing_node.is_online(
):
# If we are queued up
# detach processing node, and reassignment
# will be processed at the next tick
if self.status == status_codes.QUEUED:
logger.info(
"Processing node {} went offline, reassigning {}..."
.format(self.processing_node, self))
self.uuid = ''
self.processing_node = None
self.status = None
self.save()
elif self.status == status_codes.RUNNING:
# Task was running and processing node went offline
# It could have crashed due to low memory
# or perhaps it went offline due to network errors.
# We can't easily differentiate between the two, so we need
# to notify the user because if it crashed due to low memory
# the user might need to take action (or be stuck in an infinite loop)
raise ProcessingError(
"Processing node went offline. This could be due to insufficient memory or a network error."
)
if self.processing_node:
# Need to process some images (UUID not yet set and task doesn't have pending actions)?
if not self.uuid and self.pending_action is None and self.status is None:
logger.info("Processing... {}".format(self))
images = [
image.path() for image in self.imageupload_set.all()
]
# Track upload progress, but limit the number of DB updates
# to every 2 seconds (and always record the 100% progress)
last_update = 0
def callback(progress):
nonlocal last_update
if time.time() - last_update >= 2 or (
progress >= 1.0 - 1e-6
and progress <= 1.0 + 1e-6):
Task.objects.filter(pk=self.id).update(
upload_progress=progress)
last_update = time.time()
# This takes a while
uuid = self.processing_node.process_new_task(
images, self.name, self.options, callback)
# Refresh task object before committing change
self.refresh_from_db()
self.uuid = uuid
self.save()
# TODO: log process has started processing
if self.pending_action is not None:
if self.pending_action == pending_actions.CANCEL:
# Do we need to cancel the task on the processing node?
logger.info("Canceling {}".format(self))
if self.processing_node and self.uuid:
# Attempt to cancel the task on the processing node
# We don't care if this fails (we tried)
try:
self.processing_node.cancel_task(self.uuid)
except ProcessingException:
logger.warning(
"Could not cancel {} on processing node. We'll proceed anyway..."
.format(self))
self.status = status_codes.CANCELED
self.pending_action = None
self.save()
else:
raise ProcessingError(
"Cannot cancel a task that has no processing node or UUID"
)
elif self.pending_action == pending_actions.RESTART:
logger.info("Restarting {}".format(self))
if self.processing_node:
# Check if the UUID is still valid, as processing nodes purge
# results after a set amount of time, the UUID might have been eliminated.
uuid_still_exists = False
if self.uuid:
try:
info = self.processing_node.get_task_info(
self.uuid)
uuid_still_exists = info['uuid'] == self.uuid
except ProcessingException:
pass
need_to_reprocess = False
if uuid_still_exists:
# Good to go
try:
self.processing_node.restart_task(
self.uuid, self.options)
except ProcessingError as e:
# Something went wrong
logger.warning(
"Could not restart {}, will start a new one"
.format(self))
need_to_reprocess = True
else:
need_to_reprocess = True
if need_to_reprocess:
logger.info(
"{} needs to be reprocessed".format(self))
# Task has been purged (or processing node is offline)
# Process this as a new task
# Removing its UUID will cause the scheduler
# to process this the next tick
self.uuid = ''
# We also remove the "rerun-from" parameter if it's set
self.options = list(
filter(lambda d: d['name'] != 'rerun-from',
self.options))
self.upload_progress = 0
self.console_output = ""
self.processing_time = -1
self.status = None
self.last_error = None
self.pending_action = None
self.running_progress = 0
self.save()
else:
raise ProcessingError(
"Cannot restart a task that has no processing node"
)
elif self.pending_action == pending_actions.REMOVE:
logger.info("Removing {}".format(self))
if self.processing_node and self.uuid:
# Attempt to delete the resources on the processing node
# We don't care if this fails, as resources on processing nodes
# Are expected to be purged on their own after a set amount of time anyway
try:
self.processing_node.remove_task(self.uuid)
except ProcessingException:
pass
# What's more important is that we delete our task properly here
self.delete()
# Stop right here!
return
if self.processing_node:
# Need to update status (first time, queued or running?)
if self.uuid and self.status in [
None, status_codes.QUEUED, status_codes.RUNNING
]:
# Update task info from processing node
info = self.processing_node.get_task_info(self.uuid)
self.processing_time = info["processingTime"]
self.status = info["status"]["code"]
current_lines_count = len(self.console_output.split("\n"))
console_output = self.processing_node.get_task_console_output(
self.uuid, current_lines_count)
if len(console_output) > 0:
self.console_output += "\n".join(console_output) + '\n'
# Update running progress
for line in console_output:
for line_match, value in self.TASK_OUTPUT_MILESTONES.items(
):
if line_match in line:
self.running_progress = value
break
if "errorMessage" in info["status"]:
self.last_error = info["status"]["errorMessage"]
# Has the task just been canceled, failed, or completed?
if self.status in [
status_codes.FAILED, status_codes.COMPLETED,
status_codes.CANCELED
]:
logger.info("Processing status: {} for {}".format(
self.status, self))
if self.status == status_codes.COMPLETED:
assets_dir = self.assets_path("")
# Remove previous assets directory
if os.path.exists(assets_dir):
logger.info(
"Removing old assets directory: {} for {}".
format(assets_dir, self))
shutil.rmtree(assets_dir)
os.makedirs(assets_dir)
logger.info(
"Downloading all.zip for {}".format(self))
# Download all assets
zip_stream = self.processing_node.download_task_asset(
self.uuid, "all.zip")
zip_path = os.path.join(assets_dir, "all.zip")
with open(zip_path, 'wb') as fd:
for chunk in zip_stream.iter_content(4096):
fd.write(chunk)
logger.info(
"Done downloading all.zip for {}".format(self))
# Extract from zip
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
logger.info(
"Extracted all.zip for {}".format(self))
# Populate *_extent fields
extent_fields = [
(os.path.realpath(
self.assets_path("odm_orthophoto",
"odm_orthophoto.tif")),
'orthophoto_extent'),
(os.path.realpath(
self.assets_path("odm_dem", "dsm.tif")),
'dsm_extent'),
(os.path.realpath(
self.assets_path("odm_dem", "dtm.tif")),
'dtm_extent'),
]
for raster_path, field in extent_fields:
if os.path.exists(raster_path):
# Read extent and SRID
raster = GDALRaster(raster_path)
extent = OGRGeometry.from_bbox(
raster.extent)
# It will be implicitly transformed into the SRID of the model’s field
# self.field = GEOSGeometry(...)
setattr(
self, field,
GEOSGeometry(extent.wkt,
srid=raster.srid))
logger.info(
"Populated extent field with {} for {}"
.format(raster_path, self))
self.update_available_assets_field()
self.running_progress = 1.0
self.save()
from app.plugins import signals as plugin_signals
plugin_signals.task_completed.send_robust(
sender=self.__class__, task_id=self.id)
else:
# FAILED, CANCELED
self.save()
else:
# Still waiting...
self.save()
except ProcessingError as e:
self.set_failure(str(e))
except (ConnectionRefusedError, ConnectionError) as e:
logger.warning(
"{} cannot communicate with processing node: {}".format(
self, str(e)))
except ProcessingTimeout as e:
logger.warning(
"{} timed out with error: {}. We'll try reprocessing at the next tick."
.format(self, str(e)))
def get_tile_path(self, tile_type, z, x, y):
return self.assets_path("{}_tiles".format(tile_type), z, x,
"{}.png".format(y))
def get_tile_json_url(self, tile_type):
return "/api/projects/{}/tasks/{}/{}/tiles.json".format(
self.project.id, self.id, tile_type)
def get_map_items(self):
types = []
if 'orthophoto.tif' in self.available_assets:
types.append('orthophoto')
if 'dsm.tif' in self.available_assets: types.append('dsm')
if 'dtm.tif' in self.available_assets: types.append('dtm')
return {
'tiles': [{
'url': self.get_tile_json_url(t),
'type': t
} for t in types],
'meta': {
'task': {
'id': str(self.id),
'project': self.project.id,
'public': self.public
}
}
}
def get_model_display_params(self):
"""
Subset of a task fields used in the 3D model display view
"""
return {
'id': str(self.id),
'project': self.project.id,
'available_assets': self.available_assets,
'public': self.public
}
def generate_deferred_asset(self, archive, directory):
"""
:param archive: path of the destination .zip file (relative to /assets/ directory)
:param directory: path of the source directory to compress (relative to /assets/ directory)
:return: full path of the generated archive
"""
archive_path = self.assets_path(archive)
directory_path = self.assets_path(directory)
if not os.path.exists(directory_path):
raise FileNotFoundError("{} does not exist".format(directory_path))
if not os.path.exists(archive_path):
shutil.make_archive(
os.path.splitext(archive_path)[0], 'zip', directory_path)
return archive_path
def update_available_assets_field(self, commit=False):
"""
Updates the available_assets field with the actual types of assets available
:param commit: when True also saves the model, otherwise the user should manually call save()
"""
all_assets = list(self.ASSETS_MAP.keys())
self.available_assets = [
asset for asset in all_assets
if self.is_asset_available_slow(asset)
]
if commit: self.save()
def delete(self, using=None, keep_parents=False):
task_id = self.id
from app.plugins import signals as plugin_signals
plugin_signals.task_removing.send_robust(sender=self.__class__,
task_id=task_id)
directory_to_delete = os.path.join(
settings.MEDIA_ROOT, task_directory_path(self.id, self.project.id))
super(Task, self).delete(using, keep_parents)
# Remove files related to this task
try:
shutil.rmtree(directory_to_delete)
except FileNotFoundError as e:
logger.warning(e)
plugin_signals.task_removed.send_robust(sender=self.__class__,
task_id=task_id)
def set_failure(self, error_message):
logger.error("FAILURE FOR {}: {}".format(self, error_message))
self.last_error = error_message
self.status = status_codes.FAILED
self.pending_action = None
self.save()
def find_all_files_matching(self, regex):
directory = full_task_directory_path(self.id, self.project.id)
return [
os.path.join(directory, f) for f in os.listdir(directory)
if re.match(regex, f, re.IGNORECASE)
]
def resize_images(self):
"""
Destructively resize this task's JPG images while retaining EXIF tags.
Resulting images are always converted to JPG.
TODO: add support for tiff files
:return list containing paths of resized images and resize ratios
"""
if self.resize_to < 0:
logger.warning(
"We were asked to resize images to {}, this might be an error."
.format(self.resize_to))
return []
images_path = self.find_all_files_matching(r'.*\.jpe?g$')
total_images = len(images_path)
resized_images_count = 0
last_update = 0
def callback(retval=None):
nonlocal last_update
nonlocal resized_images_count
nonlocal total_images
resized_images_count += 1
if time.time() - last_update >= 2:
Task.objects.filter(pk=self.id).update(
resize_progress=(float(resized_images_count) /
float(total_images)))
last_update = time.time()
with ThreadPoolExecutor(max_workers=cpu_count()) as executor:
resized_images = list(
filter(
lambda i: i is not None,
executor.map(
partial(resize_image,
resize_to=self.resize_to,
done=callback), images_path)))
Task.objects.filter(pk=self.id).update(resize_progress=1.0)
return resized_images
def resize_gcp(self, resized_images):
"""
Destructively change this task's GCP file (if any)
by resizing the location of GCP entries.
:param resized_images: list of objects having "path" and "resize_ratio" keys
for example [{'path': 'path/to/DJI_0018.jpg', 'resize_ratio': 0.25}, ...]
:return: path to changed GCP file or None if no GCP file was found/changed
"""
gcp_path = self.find_all_files_matching(r'.*\.txt$')
if len(gcp_path) == 0: return None
# Assume we only have a single GCP file per task
gcp_path = gcp_path[0]
resize_script_path = os.path.join(settings.BASE_DIR, 'app', 'scripts',
'resize_gcp.js')
dict = {}
for ri in resized_images:
dict[os.path.basename(ri['path'])] = ri['resize_ratio']
try:
new_gcp_content = subprocess.check_output("node {} {} '{}'".format(
quote(resize_script_path), quote(gcp_path), json.dumps(dict)),
shell=True)
with open(gcp_path, 'w') as f:
f.write(new_gcp_content.decode('utf-8'))
logger.info("Resized GCP file {}".format(gcp_path))
return gcp_path
except subprocess.CalledProcessError as e:
logger.warning("Could not resize GCP file {}: {}".format(
gcp_path, str(e)))
return None
class Meta:
permissions = (('view_task', 'Can view task'), )
class Task(PhotoEnabled, models.Model):
"""
A task, also used to generate a todo list, i.e. those without a completed date.
Tasks need a staff member/s to do the work::
>>> from django.contrib.auth.models import User
>>> from caretaking.models import Staff
>>> darryl, created = User.objects.get_or_create(username='******', first_name='Darryl',
... last_name='Cousins')
>>> if created:
... darryl.save()
>>> caretaker, created = Staff.objects.get_or_create(user=darryl, title='Caretaker')
A type of task is also required::
>>> from caretaking.models import TaskType
>>> maintenance, created = TaskType.objects.get_or_create(name='Maintenance')
>>> if created:
... maintenance.save()
>>> duties, created = TaskType.objects.get_or_create(name='Duties')
>>> if created:
... duties.save()
Without a completed date then the task can be considered a todo::
>>> from caretaking.models import Task
>>> todo = Task.objects.create(description="Clear downpipe at library", urgency='high',
... point='POINT (172.29307 -43.75858)', staff=caretaker)
>>> todo.save()
>>> print(todo)
Clear downpipe at library
>>> todo.tasktype.add(maintenance)
>>> todo.tasktype.add(duties)
It can have more than one point of activity::
>>> todo.point = 'MULTIPOINT ((172.29307 -43.75858), (172.30 -43.76))'
>>> todo.save()
Assign the task::
>>> print(todo.staff)
Darryl Cousins (Caretaker)
It has a location::
>>> print(todo.point)
SRID=4326;MULTIPOINT (172.29307 -43.75858, 172.3 -43.76)
Maybe the job got done too::
>>> from datetime import date
>>> todo.completed = date(2017, 3, 17)
>>> todo.save()
>>> print(todo.completed)
2017-03-17
>>> print(todo)
Fri 17 Mar 2017 Clear downpipe at library
To select tasks for the caretaker::
>>> Task.objects.filter(staff=caretaker).count()
11
Accordingly we expect none from the assistant::
>>> maria, created = User.objects.get_or_create(username='******', first_name='Maria',
... last_name='Halloumis')
>>> if created:
... maria.save()
>>> assistant, created = Staff.objects.get_or_create(user=maria, title='Assistant')
>>> Task.objects.filter(staff=assistant).count()
0
We can select earliest and latest Tasks by completed date::
>>> Task.objects.filter(staff=caretaker).earliest()
<Task: Fri 10 Mar ...>
Or a count of tasks completed for the year::
>>> Task.objects.filter(staff=caretaker).filter(completed__year=2017).count()
11
"""
URGENCY = (('low', 'Low'), ('med', 'Medium'), ('high', 'High'))
taskid = models.AutoField(primary_key=True)
description = models.TextField(
blank=False,
null=False,
help_text=_("A descriptive summary of the task"))
completed = models.DateField(blank=True,
null=True,
help_text=_("The date completed"))
urgency = models.CharField(max_length=4,
choices=URGENCY,
default='low',
help_text=_("The urgency of the task"))
tasktype = models.ManyToManyField('TaskType',
help_text=_("One or more types"))
staff = models.ForeignKey('Staff',
blank=True,
null=True,
on_delete=models.DO_NOTHING,
help_text=("Staff member performing the work"))
point = GeometryField(blank=True,
null=True,
help_text=("Point or Multipoint for the task"))
class Meta:
get_latest_by = 'completed'
ordering = ('completed', )
def __str__(self):
"Returns completed date and a truncated line of the tasks description."
day = self.completed.strftime(
"%a %d %b %Y") if self.completed else None
if day:
return '{0} {1:.100}'.format(day, self.description)
else:
return '{:.100}'.format(self.description)
def get_absolute_url(self):
if self.completed:
return self.get_diary_url()
else:
return (reverse('todo-list'))
def get_diary_entry(self):
from caretaking.models.diary import Diary
diary = Diary.objects.filter(staff=self.staff).get(day=self.completed)
return diary
def get_diary_url(self):
return self.get_diary_entry().get_absolute_url()
def locations(self):
"""Find the location or locations in which the point falls"""
return Location.objects.filter(polygon__intersects=self.point).exclude(
name='CollegeBoundary')
def test_deconstruct_empty(self):
field = GeometryField()
*_, kwargs = field.deconstruct()
self.assertEqual(kwargs, {'srid': 4326})
def as_sqlite(self, compiler, connection):
function = None
geo_field = GeometryField(srid=self.srid)
if geo_field.geodetic(connection):
function = 'GeodesicLength' if self.spheroid else 'GreatCircleLength'
return super().as_sql(compiler, connection, function=function)
def test_deconstruct_empty(self):
field = GeometryField()
*_, kwargs = field.deconstruct()
self.assertEqual(kwargs, {'srid': 4326})
def __init__(self, *expressions, **extra):
if 'output_field' not in extra:
extra['output_field'] = GeometryField()
super(GeomOutputGeoFunc, self).__init__(*expressions, **extra)
def output_field(self):
return GeometryField(srid=self.geo_field.srid)
from decimal import Decimal
class Task(models.Model):
ASSETS_MAP = {
'all.zip': 'all.zip',
'orthophoto.tif': os.path.join('odm_orthophoto', 'odm_orthophoto.tif'),
'orthophoto.png': os.path.join('odm_orthophoto', 'odm_orthophoto.png'),
'orthophoto.mbtiles': os.path.join('odm_orthophoto', 'odm_orthophoto.mbtiles'),
'georeferenced_model.las': os.path.join('odm_georeferencing', 'odm_georeferenced_model.las'),
'georeferenced_model.laz': os.path.join('odm_georeferencing', 'odm_georeferenced_model.laz'),
'georeferenced_model.ply': os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply'),
'georeferenced_model.csv': os.path.join('odm_georeferencing', 'odm_georeferenced_model.csv'),
'textured_model.zip': {
'deferred_path': 'textured_model.zip',
'deferred_compress_dir': 'odm_texturing'
},
'dtm.tif': os.path.join('odm_dem', 'dtm.tif'),
'dsm.tif': os.path.join('odm_dem', 'dsm.tif'),
'dtm_tiles.zip': {
'deferred_path': 'dtm_tiles.zip',
'deferred_compress_dir': 'dtm_tiles'
},
'dsm_tiles.zip': {
'deferred_path': 'dsm_tiles.zip',
'deferred_compress_dir': 'dsm_tiles'
},
'orthophoto_tiles.zip': {
'deferred_path': 'orthophoto_tiles.zip',
'deferred_compress_dir': 'orthophoto_tiles'
},
'cameras.json': 'cameras.json',
'shots.geojson': os.path.join('odm_report', 'shots.geojson'),
}
STATUS_CODES = (
(status_codes.QUEUED, 'QUEUED'),
(status_codes.RUNNING, 'RUNNING'),
(status_codes.FAILED, 'FAILED'),
(status_codes.COMPLETED, 'COMPLETED'),
(status_codes.CANCELED, 'CANCELED'),
)
PENDING_ACTIONS = (
(pending_actions.CANCEL, 'CANCEL'),
(pending_actions.REMOVE, 'REMOVE'),
(pending_actions.RESTART, 'RESTART'),
(pending_actions.RESIZE, 'RESIZE'),
(pending_actions.IMPORT, 'IMPORT'),
)
TASK_PROGRESS_LAST_VALUE = 0.85
id = models.UUIDField(primary_key=True, default=uuid_module.uuid4, unique=True, serialize=False, editable=False, verbose_name=_("Id"))
uuid = models.CharField(max_length=255, db_index=True, default='', blank=True, help_text=_("Identifier of the task (as returned by NodeODM API)"), verbose_name=_("UUID"))
project = models.ForeignKey(Project, on_delete=models.CASCADE, help_text=_("Project that this task belongs to"), verbose_name=_("Project"))
name = models.CharField(max_length=255, null=True, blank=True, help_text=_("A label for the task"), verbose_name=_("Name"))
processing_time = models.IntegerField(default=-1, help_text=_("Number of milliseconds that elapsed since the beginning of this task (-1 indicates that no information is available)"), verbose_name=_("Processing Time"))
processing_node = models.ForeignKey(ProcessingNode, on_delete=models.SET_NULL, null=True, blank=True, help_text=_("Processing node assigned to this task (or null if this task has not been associated yet)"), verbose_name=_("Processing Node"))
auto_processing_node = models.BooleanField(default=True, help_text=_("A flag indicating whether this task should be automatically assigned a processing node"), verbose_name=_("Auto Processing Node"))
status = models.IntegerField(choices=STATUS_CODES, db_index=True, null=True, blank=True, help_text=_("Current status of the task"), verbose_name=_("Status"))
last_error = models.TextField(null=True, blank=True, help_text=_("The last processing error received"), verbose_name=_("Last Error"))
options = fields.JSONField(default=dict, blank=True, help_text=_("Options that are being used to process this task"), validators=[validate_task_options], verbose_name=_("Options"))
available_assets = fields.ArrayField(models.CharField(max_length=80), default=list, blank=True, help_text=_("List of available assets to download"), verbose_name=_("Available Assets"))
console_output = models.TextField(null=False, default="", blank=True, help_text=_("Console output of the processing node"), verbose_name=_("Console Output"))
orthophoto_extent = GeometryField(null=True, blank=True, srid=4326, help_text=_("Extent of the orthophoto"), verbose_name=_("Orthophoto Extent"))
dsm_extent = GeometryField(null=True, blank=True, srid=4326, help_text="Extent of the DSM", verbose_name=_("DSM Extent"))
dtm_extent = GeometryField(null=True, blank=True, srid=4326, help_text="Extent of the DTM", verbose_name=_("DTM Extent"))
# mission
created_at = models.DateTimeField(default=timezone.now, help_text=_("Creation date"), verbose_name=_("Created at"))
pending_action = models.IntegerField(choices=PENDING_ACTIONS, db_index=True, null=True, blank=True, help_text=_("A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration."), verbose_name=_("Pending Action"))
public = models.BooleanField(default=False, help_text=_("A flag indicating whether this task is available to the public"), verbose_name=_("Public"))
resize_to = models.IntegerField(default=-1, help_text=_("When set to a value different than -1, indicates that the images for this task have been / will be resized to the size specified here before processing."), verbose_name=_("Resize To"))
upload_progress = models.FloatField(default=0.0,
help_text=_("Value between 0 and 1 indicating the upload progress of this task's files to the processing node"),
verbose_name=_("Upload Progress"),
blank=True)
resize_progress = models.FloatField(default=0.0,
help_text=_("Value between 0 and 1 indicating the resize progress of this task's images"),
verbose_name=_("Resize Progress"),
blank=True)
running_progress = models.FloatField(default=0.0,
help_text=_("Value between 0 and 1 indicating the running progress (estimated) of this task"),
verbose_name=_("Running Progress"),
blank=True)
import_url = models.TextField(null=False, default="", blank=True, help_text=_("URL this task is imported from (only for imported tasks)"), verbose_name=_("Import URL"))
images_count = models.IntegerField(null=False, blank=True, default=0, help_text=_("Number of images associated with this task"), verbose_name=_("Images Count"))
partial = models.BooleanField(default=False, help_text=_("A flag indicating whether this task is currently waiting for information or files to be uploaded before being considered for processing."), verbose_name=_("Partial"))
class Meta:
verbose_name = _("Task")
verbose_name_plural = _("Tasks")
def __init__(self, *args, **kwargs):
super(Task, self).__init__(*args, **kwargs)
# To help keep track of changes to the project id
self.__original_project_id = self.project.id
def __str__(self):
name = self.name if self.name is not None else gettext("unnamed")
return 'Task [{}] ({})'.format(name, self.id)
def move_assets(self, old_project_id, new_project_id):
"""
Moves the task's folder, update ImageFields and orthophoto files to a new project
"""
old_task_folder = full_task_directory_path(self.id, old_project_id)
new_task_folder = full_task_directory_path(self.id, new_project_id)
new_task_folder_parent = os.path.abspath(os.path.join(new_task_folder, os.pardir))
try:
if os.path.exists(old_task_folder) and not os.path.exists(new_task_folder):
# Use parent, otherwise we get a duplicate directory in there
if not os.path.exists(new_task_folder_parent):
os.makedirs(new_task_folder_parent)
shutil.move(old_task_folder, new_task_folder_parent)
logger.info("Moved task folder from {} to {}".format(old_task_folder, new_task_folder))
with transaction.atomic():
for img in self.imageupload_set.all():
prev_name = img.image.name
img.image.name = assets_directory_path(self.id, new_project_id,
os.path.basename(img.image.name))
logger.info("Changing {} to {}".format(prev_name, img))
img.save()
else:
logger.warning("Project changed for task {}, but either {} doesn't exist, or {} already exists. This doesn't look right, so we will not move any files.".format(self,
old_task_folder,
new_task_folder))
except shutil.Error as e:
logger.warning("Could not move assets folder for task {}. We're going to proceed anyway, but you might experience issues: {}".format(self, e))
def save(self, *args, **kwargs):
if self.project.id != self.__original_project_id:
self.move_assets(self.__original_project_id, self.project.id)
self.__original_project_id = self.project.id
# Autovalidate on save
self.full_clean()
super(Task, self).save(*args, **kwargs)
def assets_path(self, *args):
"""
Get a path relative to the place where assets are stored
"""
return self.task_path("assets", *args)
def task_path(self, *args):
"""
Get path relative to the root task directory
"""
return os.path.join(settings.MEDIA_ROOT,
assets_directory_path(self.id, self.project.id, ""),
*args)
def is_asset_available_slow(self, asset):
"""
Checks whether a particular asset is available in the file system
Generally this should never be used directly, as it's slow. Use the available_assets field
in the database instead.
:param asset: one of ASSETS_MAP keys
:return: boolean
"""
if asset in self.ASSETS_MAP:
value = self.ASSETS_MAP[asset]
if isinstance(value, str):
return os.path.exists(self.assets_path(value))
elif isinstance(value, dict):
if 'deferred_compress_dir' in value:
return os.path.exists(self.assets_path(value['deferred_compress_dir']))
return False
def get_asset_download_path(self, asset):
"""
Get the path to an asset download
:param asset: one of ASSETS_MAP keys
:return: path
"""
if asset in self.ASSETS_MAP:
value = self.ASSETS_MAP[asset]
if isinstance(value, str):
return self.assets_path(value)
elif isinstance(value, dict):
if 'deferred_path' in value and 'deferred_compress_dir' in value:
return self.generate_deferred_asset(value['deferred_path'], value['deferred_compress_dir'])
else:
raise FileNotFoundError("{} is not a valid asset (invalid dict values)".format(asset))
else:
raise FileNotFoundError("{} is not a valid asset (invalid map)".format(asset))
else:
raise FileNotFoundError("{} is not a valid asset".format(asset))
def handle_import(self):
self.console_output += gettext("Importing assets...") + "\n"
self.save()
zip_path = self.assets_path("all.zip")
if self.import_url and not os.path.exists(zip_path):
try:
# TODO: this is potentially vulnerable to a zip bomb attack
# mitigated by the fact that a valid account is needed to
# import tasks
logger.info("Importing task assets from {} for {}".format(self.import_url, self))
download_stream = requests.get(self.import_url, stream=True, timeout=10)
content_length = download_stream.headers.get('content-length')
total_length = int(content_length) if content_length is not None else None
downloaded = 0
last_update = 0
with open(zip_path, 'wb') as fd:
for chunk in download_stream.iter_content(4096):
downloaded += len(chunk)
if time.time() - last_update >= 2:
# Update progress
if total_length is not None:
Task.objects.filter(pk=self.id).update(running_progress=(float(downloaded) / total_length) * 0.9)
self.check_if_canceled()
last_update = time.time()
fd.write(chunk)
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, ReadTimeoutError) as e:
raise NodeServerError(e)
self.refresh_from_db()
try:
self.extract_assets_and_complete()
except zipfile.BadZipFile:
raise NodeServerError(gettext("Invalid zip file"))
images_json = self.assets_path("images.json")
if os.path.exists(images_json):
try:
with open(images_json) as f:
images = json.load(f)
self.images_count = len(images)
except:
logger.warning("Cannot read images count from imported task {}".format(self))
pass
self.pending_action = None
self.processing_time = 0
self.save()
def process(self):
"""
This method contains the logic for processing tasks asynchronously
from a background thread or from a worker. Here tasks that are
ready to be processed execute some logic. This could be communication
with a processing node or executing a pending action.
"""
try:
if self.pending_action == pending_actions.IMPORT:
self.handle_import()
if self.pending_action == pending_actions.RESIZE:
resized_images = self.resize_images()
self.refresh_from_db()
self.resize_gcp(resized_images)
self.pending_action = None
self.save()
if self.auto_processing_node and not self.status in [status_codes.FAILED, status_codes.CANCELED]:
# No processing node assigned and need to auto assign
if self.processing_node is None:
# Assign first online node with lowest queue count
self.processing_node = ProcessingNode.find_best_available_node()
if self.processing_node:
self.processing_node.queue_count += 1 # Doesn't have to be accurate, it will get overridden later
self.processing_node.save()
logger.info("Automatically assigned processing node {} to {}".format(self.processing_node, self))
self.save()
# Processing node assigned, but is offline and no errors
if self.processing_node and not self.processing_node.is_online():
# If we are queued up
# detach processing node, and reassignment
# will be processed at the next tick
if self.status == status_codes.QUEUED:
logger.info("Processing node {} went offline, reassigning {}...".format(self.processing_node, self))
self.uuid = ''
self.processing_node = None
self.status = None
self.save()
elif self.status == status_codes.RUNNING:
# Task was running and processing node went offline
# It could have crashed due to low memory
# or perhaps it went offline due to network errors.
# We can't easily differentiate between the two, so we need
# to notify the user because if it crashed due to low memory
# the user might need to take action (or be stuck in an infinite loop)
raise NodeServerError("Processing node went offline. This could be due to insufficient memory or a network error.")
if self.processing_node:
# Need to process some images (UUID not yet set and task doesn't have pending actions)?
if not self.uuid and self.pending_action is None and self.status is None:
logger.info("Processing... {}".format(self))
images = [image.path() for image in self.imageupload_set.all()]
# Track upload progress, but limit the number of DB updates
# to every 2 seconds (and always record the 100% progress)
last_update = 0
def callback(progress):
nonlocal last_update
time_has_elapsed = time.time() - last_update >= 2
if time_has_elapsed:
testWatch.manual_log_call("Task.process.callback")
self.check_if_canceled()
Task.objects.filter(pk=self.id).update(upload_progress=float(progress) / 100.0)
last_update = time.time()
# This takes a while
try:
uuid = self.processing_node.process_new_task(images, self.name, self.options, callback)
except NodeConnectionError as e:
# If we can't create a task because the node is offline
# We want to fail instead of trying again
raise NodeServerError(gettext('Connection error: %(error)s') % {'error': str(e)})
# Refresh task object before committing change
self.refresh_from_db()
self.upload_progress = 1.0
self.uuid = uuid
self.save()
# TODO: log process has started processing
if self.pending_action is not None:
if self.pending_action == pending_actions.CANCEL:
# Do we need to cancel the task on the processing node?
logger.info("Canceling {}".format(self))
if self.processing_node and self.uuid:
# Attempt to cancel the task on the processing node
# We don't care if this fails (we tried)
try:
self.processing_node.cancel_task(self.uuid)
except OdmError:
logger.warning("Could not cancel {} on processing node. We'll proceed anyway...".format(self))
self.status = status_codes.CANCELED
self.pending_action = None
self.save()
else:
# Tasks with no processing node or UUID need no special action
self.status = status_codes.CANCELED
self.pending_action = None
self.save()
elif self.pending_action == pending_actions.RESTART:
logger.info("Restarting {}".format(self))
if self.processing_node:
# Check if the UUID is still valid, as processing nodes purge
# results after a set amount of time, the UUID might have been eliminated.
uuid_still_exists = False
if self.uuid:
try:
info = self.processing_node.get_task_info(self.uuid)
uuid_still_exists = info.uuid == self.uuid
except OdmError:
pass
need_to_reprocess = False
if uuid_still_exists:
# Good to go
try:
self.processing_node.restart_task(self.uuid, self.options)
except (NodeServerError, NodeResponseError) as e:
# Something went wrong
logger.warning("Could not restart {}, will start a new one".format(self))
need_to_reprocess = True
else:
need_to_reprocess = True
if need_to_reprocess:
logger.info("{} needs to be reprocessed".format(self))
# Task has been purged (or processing node is offline)
# Process this as a new task
# Removing its UUID will cause the scheduler
# to process this the next tick
self.uuid = ''
# We also remove the "rerun-from" parameter if it's set
self.options = list(filter(lambda d: d['name'] != 'rerun-from', self.options))
self.upload_progress = 0
self.console_output = ""
self.processing_time = -1
self.status = None
self.last_error = None
self.pending_action = None
self.running_progress = 0
self.save()
else:
raise NodeServerError(gettext("Cannot restart a task that has no processing node"))
elif self.pending_action == pending_actions.REMOVE:
logger.info("Removing {}".format(self))
if self.processing_node and self.uuid:
# Attempt to delete the resources on the processing node
# We don't care if this fails, as resources on processing nodes
# Are expected to be purged on their own after a set amount of time anyway
try:
self.processing_node.remove_task(self.uuid)
except OdmError:
pass
# What's more important is that we delete our task properly here
self.delete()
# Stop right here!
return
if self.processing_node:
# Need to update status (first time, queued or running?)
if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]:
# Update task info from processing node
if not self.console_output:
current_lines_count = 0
else:
current_lines_count = len(self.console_output.split("\n"))
info = self.processing_node.get_task_info(self.uuid, current_lines_count)
self.processing_time = info.processing_time
self.status = info.status.value
if len(info.output) > 0:
self.console_output += "\n".join(info.output) + '\n'
# Update running progress
self.running_progress = (info.progress / 100.0) * self.TASK_PROGRESS_LAST_VALUE
if info.last_error != "":
self.last_error = info.last_error
# Has the task just been canceled, failed, or completed?
if self.status in [status_codes.FAILED, status_codes.COMPLETED, status_codes.CANCELED]:
logger.info("Processing status: {} for {}".format(self.status, self))
if self.status == status_codes.COMPLETED:
assets_dir = self.assets_path("")
# Remove previous assets directory
if os.path.exists(assets_dir):
logger.info("Removing old assets directory: {} for {}".format(assets_dir, self))
shutil.rmtree(assets_dir)
os.makedirs(assets_dir)
# Download and try to extract results up to 4 times
# (~5% of the times, on large downloads, the archive could be corrupted)
retry_num = 0
extracted = False
last_update = 0
def callback(progress):
nonlocal last_update
time_has_elapsed = time.time() - last_update >= 2
if time_has_elapsed or int(progress) == 100:
Task.objects.filter(pk=self.id).update(running_progress=(
self.TASK_PROGRESS_LAST_VALUE + (float(progress) / 100.0) * 0.1))
last_update = time.time()
while not extracted:
last_update = 0
logger.info("Downloading all.zip for {}".format(self))
# Download all assets
zip_path = self.processing_node.download_task_assets(self.uuid, assets_dir, progress_callback=callback, parallel_downloads=max(1, int(16 / (2 ** retry_num))))
# Rename to all.zip
all_zip_path = self.assets_path("all.zip")
os.rename(zip_path, all_zip_path)
logger.info("Extracting all.zip for {}".format(self))
try:
self.extract_assets_and_complete()
extracted = True
except zipfile.BadZipFile:
if retry_num < 5:
logger.warning("{} seems corrupted. Retrying...".format(all_zip_path))
retry_num += 1
os.remove(all_zip_path)
else:
raise NodeServerError(gettext("Invalid zip file"))
else:
# FAILED, CANCELED
self.save()
else:
# Still waiting...
self.save()
except (NodeServerError, NodeResponseError) as e:
self.set_failure(str(e))
except NodeConnectionError as e:
logger.warning("{} connection/timeout error: {}. We'll try reprocessing at the next tick.".format(self, str(e)))
except TaskInterruptedException as e:
# Task was interrupted during image resize / upload
logger.warning("{} interrupted".format(self, str(e)))
def extract_assets_and_complete(self):
"""
Extracts assets/all.zip, populates task fields where required and assure COGs
It will raise a zipfile.BadZipFile exception is the archive is corrupted.
:return:
"""
assets_dir = self.assets_path("")
zip_path = self.assets_path("all.zip")
# Extract from zip
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
logger.info("Extracted all.zip for {}".format(self))
# Populate *_extent fields
extent_fields = [
(os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")),
'orthophoto_extent'),
(os.path.realpath(self.assets_path("odm_dem", "dsm.tif")),
'dsm_extent'),
(os.path.realpath(self.assets_path("odm_dem", "dtm.tif")),
'dtm_extent'),
]
for raster_path, field in extent_fields:
if os.path.exists(raster_path):
# Make sure this is a Cloud Optimized GeoTIFF
# if not, it will be created
try:
assure_cogeo(raster_path)
except IOError as e:
logger.warning("Cannot create Cloud Optimized GeoTIFF for %s (%s). This will result in degraded visualization performance." % (raster_path, str(e)))
# Read extent and SRID
raster = GDALRaster(raster_path)
extent = OGRGeometry.from_bbox(raster.extent)
# Make sure PostGIS supports it
with connection.cursor() as cursor:
cursor.execute("SELECT SRID FROM spatial_ref_sys WHERE SRID = %s", [raster.srid])
if cursor.rowcount == 0:
raise NodeServerError(gettext("Unsupported SRS %(code)s. Please make sure you picked a supported SRS.") % {'code': str(raster.srid)})
# It will be implicitly transformed into the SRID of the model’s field
# self.field = GEOSGeometry(...)
setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid))
logger.info("Populated extent field with {} for {}".format(raster_path, self))
self.update_available_assets_field()
self.running_progress = 1.0
self.console_output += gettext("Done!") + "\n"
self.status = status_codes.COMPLETED
self.save()
from app.plugins import signals as plugin_signals
plugin_signals.task_completed.send_robust(sender=self.__class__, task_id=self.id)
def get_tile_path(self, tile_type, z, x, y):
return self.assets_path("{}_tiles".format(tile_type), z, x, "{}.png".format(y))
def get_tile_base_url(self, tile_type):
# plant is just a special case of orthophoto
if tile_type == 'plant':
tile_type = 'orthophoto'
return "/api/projects/{}/tasks/{}/{}/".format(self.project.id, self.id, tile_type)
def get_map_items(self):
types = []
if 'orthophoto.tif' in self.available_assets: types.append('orthophoto')
if 'orthophoto.tif' in self.available_assets: types.append('plant')
if 'dsm.tif' in self.available_assets: types.append('dsm')
if 'dtm.tif' in self.available_assets: types.append('dtm')
camera_shots = ''
if 'shots.geojson' in self.available_assets: camera_shots = '/api/projects/{}/tasks/{}/download/shots.geojson'.format(self.project.id, self.id)
return {
'tiles': [{'url': self.get_tile_base_url(t), 'type': t} for t in types],
'meta': {
'task': {
'id': str(self.id),
'project': self.project.id,
'public': self.public,
'camera_shots': camera_shots
}
}
}
def get_model_display_params(self):
"""
Subset of a task fields used in the 3D model display view
"""
return {
'id': str(self.id),
'project': self.project.id,
'available_assets': self.available_assets,
'public': self.public
}
def generate_deferred_asset(self, archive, directory):
"""
:param archive: path of the destination .zip file (relative to /assets/ directory)
:param directory: path of the source directory to compress (relative to /assets/ directory)
:return: full path of the generated archive
"""
archive_path = self.assets_path(archive)
directory_path = self.assets_path(directory)
if not os.path.exists(directory_path):
raise FileNotFoundError("{} does not exist".format(directory_path))
if not os.path.exists(archive_path):
shutil.make_archive(os.path.splitext(archive_path)[0], 'zip', directory_path)
return archive_path
def update_available_assets_field(self, commit=False):
"""
Updates the available_assets field with the actual types of assets available
:param commit: when True also saves the model, otherwise the user should manually call save()
"""
all_assets = list(self.ASSETS_MAP.keys())
self.available_assets = [asset for asset in all_assets if self.is_asset_available_slow(asset)]
if commit: self.save()
def delete(self, using=None, keep_parents=False):
task_id = self.id
from app.plugins import signals as plugin_signals
plugin_signals.task_removing.send_robust(sender=self.__class__, task_id=task_id)
directory_to_delete = os.path.join(settings.MEDIA_ROOT,
task_directory_path(self.id, self.project.id))
super(Task, self).delete(using, keep_parents)
# Remove files related to this task
try:
shutil.rmtree(directory_to_delete)
except FileNotFoundError as e:
logger.warning(e)
plugin_signals.task_removed.send_robust(sender=self.__class__, task_id=task_id)
def set_failure(self, error_message):
logger.error("FAILURE FOR {}: {}".format(self, error_message))
self.last_error = error_message
self.status = status_codes.FAILED
self.pending_action = None
self.save()
def find_all_files_matching(self, regex):
directory = full_task_directory_path(self.id, self.project.id)
return [os.path.join(directory, f) for f in os.listdir(directory) if
re.match(regex, f, re.IGNORECASE)]
def check_if_canceled(self):
# Check if task has been canceled/removed
if Task.objects.only("pending_action").get(pk=self.id).pending_action in [pending_actions.CANCEL,
pending_actions.REMOVE]:
raise TaskInterruptedException()
def resize_images(self):
"""
Destructively resize this task's JPG images while retaining EXIF tags.
Resulting images are always converted to JPG.
TODO: add support for tiff files
:return list containing paths of resized images and resize ratios
"""
if self.resize_to < 0:
logger.warning("We were asked to resize images to {}, this might be an error.".format(self.resize_to))
return []
images_path = self.find_all_files_matching(r'.*\.(jpe?g|tiff?)$')
total_images = len(images_path)
resized_images_count = 0
last_update = 0
def callback(retval=None):
nonlocal last_update
nonlocal resized_images_count
nonlocal total_images
resized_images_count += 1
if time.time() - last_update >= 2:
# Update progress
Task.objects.filter(pk=self.id).update(resize_progress=(float(resized_images_count) / float(total_images)))
self.check_if_canceled()
last_update = time.time()
resized_images = list(map(partial(resize_image, resize_to=self.resize_to, done=callback), images_path))
Task.objects.filter(pk=self.id).update(resize_progress=1.0)
return resized_images
def resize_gcp(self, resized_images):
"""
Destructively change this task's GCP file (if any)
by resizing the location of GCP entries.
:param resized_images: list of objects having "path" and "resize_ratio" keys
for example [{'path': 'path/to/DJI_0018.jpg', 'resize_ratio': 0.25}, ...]
:return: path to changed GCP file or None if no GCP file was found/changed
"""
gcp_path = self.find_all_files_matching(r'.*\.txt$')
if len(gcp_path) == 0: return None
# Assume we only have a single GCP file per task
gcp_path = gcp_path[0]
image_ratios = {}
for ri in resized_images:
image_ratios[os.path.basename(ri['path']).lower()] = ri['resize_ratio']
try:
gcpFile = GCPFile(gcp_path)
gcpFile.create_resized_copy(gcp_path, image_ratios)
logger.info("Resized GCP file {}".format(gcp_path))
return gcp_path
except Exception as e:
logger.warning("Could not resize GCP file {}: {}".format(gcp_path, str(e)))
def create_task_directories(self):
"""
Create directories for this task (if they don't exist already)
"""
assets_dir = self.assets_path("")
try:
os.makedirs(assets_dir)
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(assets_dir):
pass
else:
raise
class Task(models.Model):
ASSETS_MAP = {
'all.zip': 'all.zip',
'orthophoto.tif': os.path.join('odm_orthophoto', 'odm_orthophoto.tif'),
'orthophoto.png': os.path.join('odm_orthophoto', 'odm_orthophoto.png'),
'georeferenced_model.las': os.path.join('odm_georeferencing', 'odm_georeferenced_model.las'),
'georeferenced_model.ply': os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply'),
'georeferenced_model.csv': os.path.join('odm_georeferencing', 'odm_georeferenced_model.csv'),
'textured_model.zip': {
'deferred_path': 'textured_model.zip',
'deferred_compress_dir': 'odm_texturing'
},
'dtm.tif': os.path.join('odm_dem', 'dtm.tif'),
'dsm.tif': os.path.join('odm_dem', 'dsm.tif'),
}
STATUS_CODES = (
(status_codes.QUEUED, 'QUEUED'),
(status_codes.RUNNING, 'RUNNING'),
(status_codes.FAILED, 'FAILED'),
(status_codes.COMPLETED, 'COMPLETED'),
(status_codes.CANCELED, 'CANCELED'),
)
PENDING_ACTIONS = (
(pending_actions.CANCEL, 'CANCEL'),
(pending_actions.REMOVE, 'REMOVE'),
(pending_actions.RESTART, 'RESTART'),
)
id = models.UUIDField(primary_key=True, default=uuid_module.uuid4, unique=True, serialize=False, editable=False)
uuid = models.CharField(max_length=255, db_index=True, default='', blank=True, help_text="Identifier of the task (as returned by OpenDroneMap's REST API)")
project = models.ForeignKey(Project, on_delete=models.CASCADE, help_text="Project that this task belongs to")
name = models.CharField(max_length=255, null=True, blank=True, help_text="A label for the task")
processing_lock = models.BooleanField(default=False, help_text="A flag indicating whether this task is currently locked for processing. When this flag is turned on, the task is in the middle of a processing step.")
processing_time = models.IntegerField(default=-1, help_text="Number of milliseconds that elapsed since the beginning of this task (-1 indicates that no information is available)")
processing_node = models.ForeignKey(ProcessingNode, null=True, blank=True, help_text="Processing node assigned to this task (or null if this task has not been associated yet)")
auto_processing_node = models.BooleanField(default=True, help_text="A flag indicating whether this task should be automatically assigned a processing node")
status = models.IntegerField(choices=STATUS_CODES, db_index=True, null=True, blank=True, help_text="Current status of the task")
last_error = models.TextField(null=True, blank=True, help_text="The last processing error received")
options = fields.JSONField(default=dict(), blank=True, help_text="Options that are being used to process this task", validators=[validate_task_options])
available_assets = fields.ArrayField(models.CharField(max_length=80), default=list(), blank=True, help_text="List of available assets to download")
console_output = models.TextField(null=False, default="", blank=True, help_text="Console output of the OpenDroneMap's process")
ground_control_points = models.FileField(null=True, blank=True, upload_to=gcp_directory_path, help_text="Optional Ground Control Points file to use for processing")
orthophoto_extent = GeometryField(null=True, blank=True, srid=4326, help_text="Extent of the orthophoto created by OpenDroneMap")
dsm_extent = GeometryField(null=True, blank=True, srid=4326, help_text="Extent of the DSM created by OpenDroneMap")
dtm_extent = GeometryField(null=True, blank=True, srid=4326, help_text="Extent of the DTM created by OpenDroneMap")
# mission
created_at = models.DateTimeField(default=timezone.now, help_text="Creation date")
pending_action = models.IntegerField(choices=PENDING_ACTIONS, db_index=True, null=True, blank=True, help_text="A requested action to be performed on the task. The selected action will be performed by the scheduler at the next iteration.")
public = models.BooleanField(default=False, help_text="A flag indicating whether this task is available to the public")
def __init__(self, *args, **kwargs):
super(Task, self).__init__(*args, **kwargs)
# To help keep track of changes to the project id
self.__original_project_id = self.project.id
def __str__(self):
name = self.name if self.name is not None else "unnamed"
return 'Task [{}] ({})'.format(name, self.id)
def move_assets(self, old_project_id, new_project_id):
"""
Moves the task's folder, update ImageFields and orthophoto files to a new project
"""
old_task_folder = full_task_directory_path(self.id, old_project_id)
new_task_folder = full_task_directory_path(self.id, new_project_id)
new_task_folder_parent = os.path.abspath(os.path.join(new_task_folder, os.pardir))
try:
if os.path.exists(old_task_folder) and not os.path.exists(new_task_folder):
# Use parent, otherwise we get a duplicate directory in there
if not os.path.exists(new_task_folder_parent):
os.makedirs(new_task_folder_parent)
shutil.move(old_task_folder, new_task_folder_parent)
logger.info("Moved task folder from {} to {}".format(old_task_folder, new_task_folder))
with transaction.atomic():
for img in self.imageupload_set.all():
prev_name = img.image.name
img.image.name = assets_directory_path(self.id, new_project_id,
os.path.basename(img.image.name))
logger.info("Changing {} to {}".format(prev_name, img))
img.save()
else:
logger.warning("Project changed for task {}, but either {} doesn't exist, or {} already exists. This doesn't look right, so we will not move any files.".format(self,
old_task_folder,
new_task_folder))
except shutil.Error as e:
logger.warning("Could not move assets folder for task {}. We're going to proceed anyway, but you might experience issues: {}".format(self, e))
def save(self, *args, **kwargs):
if self.project.id != self.__original_project_id:
self.move_assets(self.__original_project_id, self.project.id)
self.__original_project_id = self.project.id
# Autovalidate on save
self.full_clean()
super(Task, self).save(*args, **kwargs)
def assets_path(self, *args):
"""
Get a path relative to the place where assets are stored
"""
return os.path.join(settings.MEDIA_ROOT,
assets_directory_path(self.id, self.project.id, ""),
"assets",
*args)
def is_asset_available_slow(self, asset):
"""
Checks whether a particular asset is available in the file system
Generally this should never be used directly, as it's slow. Use the available_assets field
in the database instead.
:param asset: one of ASSETS_MAP keys
:return: boolean
"""
if asset in self.ASSETS_MAP:
value = self.ASSETS_MAP[asset]
if isinstance(value, str):
return os.path.exists(self.assets_path(value))
elif isinstance(value, dict):
if 'deferred_compress_dir' in value:
return os.path.exists(self.assets_path(value['deferred_compress_dir']))
return False
def get_asset_download_path(self, asset):
"""
Get the path to an asset download
:param asset: one of ASSETS_MAP keys
:return: path
"""
if asset in self.ASSETS_MAP:
value = self.ASSETS_MAP[asset]
if isinstance(value, str):
return self.assets_path(value)
elif isinstance(value, dict):
if 'deferred_path' in value and 'deferred_compress_dir' in value:
return self.generate_deferred_asset(value['deferred_path'], value['deferred_compress_dir'])
else:
raise FileNotFoundError("{} is not a valid asset (invalid dict values)".format(asset))
else:
raise FileNotFoundError("{} is not a valid asset (invalid map)".format(asset))
else:
raise FileNotFoundError("{} is not a valid asset".format(asset))
def process(self):
"""
This method contains the logic for processing tasks asynchronously
from a background thread or from the scheduler. Here tasks that are
ready to be processed execute some logic. This could be communication
with a processing node or executing a pending action.
"""
try:
if self.auto_processing_node and not self.status in [status_codes.FAILED, status_codes.CANCELED]:
# No processing node assigned and need to auto assign
if self.processing_node is None:
# Assign first online node with lowest queue count
self.processing_node = ProcessingNode.find_best_available_node()
if self.processing_node:
self.processing_node.queue_count += 1 # Doesn't have to be accurate, it will get overridden later
self.processing_node.save()
logger.info("Automatically assigned processing node {} to {}".format(self.processing_node, self))
self.save()
# Processing node assigned, but is offline and no errors
if self.processing_node and not self.processing_node.is_online():
# Detach processing node, will be processed at the next tick
logger.info("Processing node {} went offline, reassigning {}...".format(self.processing_node, self))
self.uuid = ''
self.processing_node = None
self.save()
if self.processing_node:
# Need to process some images (UUID not yet set and task doesn't have pending actions)?
if not self.uuid and self.pending_action is None and self.status is None:
logger.info("Processing... {}".format(self))
images = [image.path() for image in self.imageupload_set.all()]
# This takes a while
uuid = self.processing_node.process_new_task(images, self.name, self.options)
# Refresh task object before committing change
self.refresh_from_db()
self.uuid = uuid
self.save()
# TODO: log process has started processing
if self.pending_action is not None:
if self.pending_action == pending_actions.CANCEL:
# Do we need to cancel the task on the processing node?
logger.info("Canceling {}".format(self))
if self.processing_node and self.uuid:
# Attempt to cancel the task on the processing node
# We don't care if this fails (we tried)
try:
self.processing_node.cancel_task(self.uuid)
self.status = None
except ProcessingException:
logger.warning("Could not cancel {} on processing node. We'll proceed anyway...".format(self))
self.status = status_codes.CANCELED
self.pending_action = None
self.save()
else:
raise ProcessingError("Cannot cancel a task that has no processing node or UUID")
elif self.pending_action == pending_actions.RESTART:
logger.info("Restarting {}".format(self))
if self.processing_node:
# Check if the UUID is still valid, as processing nodes purge
# results after a set amount of time, the UUID might have eliminated.
uuid_still_exists = False
if self.uuid:
try:
info = self.processing_node.get_task_info(self.uuid)
uuid_still_exists = info['uuid'] == self.uuid
except ProcessingException:
pass
if uuid_still_exists:
# Good to go
try:
self.processing_node.restart_task(self.uuid)
except ProcessingError as e:
# Something went wrong
logger.warning("Could not restart {}, will start a new one".format(self))
self.uuid = ''
else:
# Task has been purged (or processing node is offline)
# Process this as a new task
# Removing its UUID will cause the scheduler
# to process this the next tick
self.uuid = ''
self.console_output = ""
self.processing_time = -1
self.status = None
self.last_error = None
self.pending_action = None
self.save()
else:
raise ProcessingError("Cannot restart a task that has no processing node")
elif self.pending_action == pending_actions.REMOVE:
logger.info("Removing {}".format(self))
if self.processing_node and self.uuid:
# Attempt to delete the resources on the processing node
# We don't care if this fails, as resources on processing nodes
# Are expected to be purged on their own after a set amount of time anyway
try:
self.processing_node.remove_task(self.uuid)
except ProcessingException:
pass
# What's more important is that we delete our task properly here
self.delete()
# Stop right here!
return
if self.processing_node:
# Need to update status (first time, queued or running?)
if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]:
# Update task info from processing node
info = self.processing_node.get_task_info(self.uuid)
self.processing_time = info["processingTime"]
self.status = info["status"]["code"]
current_lines_count = len(self.console_output.split("\n")) - 1
self.console_output += self.processing_node.get_task_console_output(self.uuid, current_lines_count)
if "errorMessage" in info["status"]:
self.last_error = info["status"]["errorMessage"]
# Has the task just been canceled, failed, or completed?
if self.status in [status_codes.FAILED, status_codes.COMPLETED, status_codes.CANCELED]:
logger.info("Processing status: {} for {}".format(self.status, self))
if self.status == status_codes.COMPLETED:
assets_dir = self.assets_path("")
# Remove previous assets directory
if os.path.exists(assets_dir):
logger.info("Removing old assets directory: {} for {}".format(assets_dir, self))
shutil.rmtree(assets_dir)
os.makedirs(assets_dir)
logger.info("Downloading all.zip for {}".format(self))
# Download all assets
zip_stream = self.processing_node.download_task_asset(self.uuid, "all.zip")
zip_path = os.path.join(assets_dir, "all.zip")
with open(zip_path, 'wb') as fd:
for chunk in zip_stream.iter_content(4096):
fd.write(chunk)
logger.info("Done downloading all.zip for {}".format(self))
# Extract from zip
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
logger.info("Extracted all.zip for {}".format(self))
# Populate *_extent fields
extent_fields = [
(os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")),
'orthophoto_extent'),
(os.path.realpath(self.assets_path("odm_dem", "dsm.tif")),
'dsm_extent'),
(os.path.realpath(self.assets_path("odm_dem", "dtm.tif")),
'dtm_extent'),
]
for raster_path, field in extent_fields:
if os.path.exists(raster_path):
# Read extent and SRID
raster = GDALRaster(raster_path)
extent = OGRGeometry.from_bbox(raster.extent)
# It will be implicitly transformed into the SRID of the model’s field
# self.field = GEOSGeometry(...)
setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid))
logger.info("Populated extent field with {} for {}".format(raster_path, self))
self.update_available_assets_field()
self.save()
else:
# FAILED, CANCELED
self.save()
else:
# Still waiting...
self.save()
except ProcessingError as e:
self.set_failure(str(e))
except (ConnectionRefusedError, ConnectionError) as e:
logger.warning("{} cannot communicate with processing node: {}".format(self, str(e)))
except ProcessingTimeout as e:
logger.warning("{} timed out with error: {}. We'll try reprocessing at the next tick.".format(self, str(e)))
def get_tile_path(self, tile_type, z, x, y):
return self.assets_path("{}_tiles".format(tile_type), z, x, "{}.png".format(y))
def get_tile_json_url(self, tile_type):
return "/api/projects/{}/tasks/{}/{}/tiles.json".format(self.project.id, self.id, tile_type)
def get_map_items(self):
types = []
if 'orthophoto.tif' in self.available_assets: types.append('orthophoto')
if 'dsm.tif' in self.available_assets: types.append('dsm')
if 'dtm.tif' in self.available_assets: types.append('dtm')
return {
'tiles': [{'url': self.get_tile_json_url(t), 'type': t} for t in types],
'meta': {
'task': {
'id': str(self.id),
'project': self.project.id,
'public': self.public
}
}
}
def get_model_display_params(self):
"""
Subset of a task fields used in the 3D model display view
"""
return {
'id': str(self.id),
'project': self.project.id,
'available_assets': self.available_assets,
'public': self.public
}
def generate_deferred_asset(self, archive, directory):
"""
:param archive: path of the destination .zip file (relative to /assets/ directory)
:param directory: path of the source directory to compress (relative to /assets/ directory)
:return: full path of the generated archive
"""
archive_path = self.assets_path(archive)
directory_path = self.assets_path(directory)
if not os.path.exists(directory_path):
raise FileNotFoundError("{} does not exist".format(directory_path))
if not os.path.exists(archive_path):
shutil.make_archive(os.path.splitext(archive_path)[0], 'zip', directory_path)
return archive_path
def update_available_assets_field(self, commit=False):
"""
Updates the available_assets field with the actual types of assets available
:param commit: when True also saves the model, otherwise the user should manually call save()
"""
all_assets = list(self.ASSETS_MAP.keys())
self.available_assets = [asset for asset in all_assets if self.is_asset_available_slow(asset)]
if commit: self.save()
def delete(self, using=None, keep_parents=False):
directory_to_delete = os.path.join(settings.MEDIA_ROOT,
task_directory_path(self.id, self.project.id))
super(Task, self).delete(using, keep_parents)
# Remove files related to this task
try:
shutil.rmtree(directory_to_delete)
except FileNotFoundError as e:
logger.warning(e)
def set_failure(self, error_message):
logger.error("FAILURE FOR {}: {}".format(self, error_message))
self.last_error = error_message
self.status = status_codes.FAILED
self.pending_action = None
self.save()
class Meta:
permissions = (
('view_task', 'Can view task'),
)
