class Base(Model):
created = DateTimeField(auto_now_add=True, null=True)
modified = DateTimeField(auto_now=True, null=True)
class Meta:
abstract = True
def __str__(self):
try:
for propname in ["name", "key", "token", "text"]:
if hasattr(self, propname):
text = getattr(self, propname).encode("utf-8")
if len(text) > 20:
return text[:20] + "..."
else:
return text
else:
return str(self.id)
except:
return str(self.id)
def update(self, d):
save = False
for k, v in d.items():
if getattr(self, k) != v:
save = True
setattr(self, k, v)
if save:
self.save()
class Order(Base):
complete = BooleanField(default=False)
duration = IntegerField(null=True) # how long it took to process the order
edited = BooleanField(default=False) # tells you whether they opened it for editing... not whether any actual edits were made
end = DateTimeField(null=True)
end_user_timezone = CharField(max_length=20, null=True)
map_format = CharField(max_length=20, null=True)
open_source = BooleanField(default=False) #is this map open-sourced, such that it can be included in open source training data?
start = DateTimeField(auto_now_add=True, null=True) # it will never be null, but have to do this because migration asks for default otherwise
style = ForeignKey("MapStyle", null=True, on_delete=SET_NULL)
token = CharField(max_length=200, null=True, unique=True) # the random string that's used to find the order in the maps
url = URLField(null=True, max_length=1000, unique=True) # URL if started from url or iframe embeded on a webpage
def __str__(self):
return self.token
def d(self):
self.delete_map()
self.delete()
def delete_map(self):
rmtree("/home/usrfd/maps/" + self.token)
def finish(self):
self.complete = True
self.end = end = datetime.now().replace(tzinfo=utc)
self.duration = (end - self.start).total_seconds()
self.save()
class WithExclusivity(Model):
has_exclusivity = BooleanField('exclusivitate', default=False)
contract = CharField(max_length=200, blank=True, default=None)
validity_from = DateTimeField('valabilitate de la', blank=True, default=None)
validity_up_to = DateTimeField('până la', blank=True, default=None)
class Meta:
abstract = True
class EntityMediaBase(EntityBase):
name = CharField(max_length=256)
uploader = ForeignKey(User, on_delete=PROTECT)
upload_datetime = DateTimeField()
md5 = SlugField(max_length=32)
""" md5 hash of the originally uploaded file. """
file = FileField()
last_edit_start = DateTimeField(null=True, blank=True)
""" Start datetime of a session in which the media's annotations were edited.
"""
last_edit_end = DateTimeField(null=True, blank=True)
""" End datetime of a session in which the media's annotations were edited.
class User(AbstractUser):
middle_initial = CharField(max_length=1)
initials = CharField(max_length=3)
organization = ForeignKey(Organization, on_delete=SET_NULL, null=True, blank=True)
last_login = DateTimeField(null=True, blank=True)
last_failed_login = DateTimeField(null=True, blank=True)
failed_login_count = IntegerField(default=0)
def __str__(self):
if self.first_name or self.last_name:
return f"{self.first_name} {self.last_name}"
else:
return "---"
class AlgorithmResult(Model):
algorithm = ForeignKey(Algorithm, on_delete=CASCADE)
user = ForeignKey(User, on_delete=CASCADE)
media = ManyToManyField(EntityMediaBase)
started = DateTimeField()
stopped = DateTimeField()
result = EnumField(JobResult)
message = CharField(max_length=128)
setup_log = FileField(null=True, blank=True)
algorithm_log = FileField(null=True, blank=True)
teardown_log = FileField(null=True, blank=True)
def __str__(self):
return f"{self.algorithm.name}, {self.result}, started {self.started}"
class TemporaryFile(Model):
""" Represents a temporary file in the system, can be used for algorithm results or temporary outputs """
name = CharField(max_length=128)
""" Human readable name for display purposes """
project = ForeignKey(Project, on_delete=CASCADE)
""" Project the temporary file resides in """
user = ForeignKey(User, on_delete=PROTECT)
""" User who created the temporary file """
path = FilePathField(path=settings.MEDIA_ROOT, null=True, blank=True)
""" Path to file on storage """
lookup = SlugField(max_length=32)
""" unique lookup (md5sum of something useful) """
created_datetime = DateTimeField()
""" Time that the file was created """
eol_datetime = DateTimeField()
""" Time the file expires (reaches EoL) """
def expire(self):
""" Set a given temporary file as expired """
past = datetime.datetime.utcnow() - datetime.timedelta(hours=1)
past = pytz.timezone("UTC").localize(past)
self.eol_datetime = past
self.save()
def from_local(path, name, project, user, lookup, hours, is_upload=False):
""" Given a local file create a temporary file storage object
:returns A saved TemporaryFile:
"""
extension = os.path.splitext(name)[-1]
destination_fp = os.path.join(settings.MEDIA_ROOT, f"{project.id}",
f"{uuid.uuid1()}{extension}")
os.makedirs(os.path.dirname(destination_fp), exist_ok=True)
if is_upload:
download_uploaded_file(path, user, destination_fp)
else:
shutil.copyfile(path, destination_fp)
now = datetime.datetime.utcnow()
eol = now + datetime.timedelta(hours=hours)
temp_file = TemporaryFile(name=name,
project=project,
user=user,
path=destination_fp,
lookup=lookup,
created_datetime=now,
eol_datetime=eol)
temp_file.save()
return temp_file
class Project(Model):
name = CharField(max_length=128)
creator = ForeignKey(User, on_delete=PROTECT, related_name='creator')
created = DateTimeField(auto_now_add=True)
size = BigIntegerField(default=0)
"""Size of all media in project in bytes.
"""
num_files = IntegerField(default=0)
summary = CharField(max_length=1024)
filter_autocomplete = JSONField(null=True, blank=True)
section_order = ArrayField(CharField(max_length=128), default=list)
def has_user(self, user_id):
return self.membership_set.filter(user_id=user_id).exists()
def user_permission(self, user_id):
permission = None
qs = self.membership_set.filter(user_id=user_id)
if qs.exists():
permission = qs[0].permission
return permission
def __str__(self):
return self.name
def delete(self, *args, **kwargs):
# Delete attribute types
AttributeTypeBase.objects.filter(project=self).delete()
# Delete entities
qs = EntityBase.objects.filter(project=self)
delete_polymorphic_qs(qs)
# Delete entity types
qs = EntityTypeBase.objects.filter(project=self)
delete_polymorphic_qs(qs)
super().delete(*args, **kwargs)
class Project(Model):
name = CharField(max_length=128)
creator = ForeignKey(User, on_delete=PROTECT, related_name='creator')
created = DateTimeField(auto_now_add=True)
size = BigIntegerField(default=0)
"""Size of all media in project in bytes.
"""
num_files = IntegerField(default=0)
summary = CharField(max_length=1024)
filter_autocomplete = JSONField(null=True, blank=True)
def has_user(self, user_id):
return self.membership_set.filter(user_id=user_id).exists()
def user_permission(self, user_id):
permission = None
qs = self.membership_set.filter(user_id=user_id)
if qs.exists():
permission = qs[0].permission
return permission
def __str__(self):
return self.name
def delete(self, *args, **kwargs):
Version.objects.filter(project=self).delete()
MediaType.objects.filter(project=self).delete()
LocalizationType.objects.filter(project=self).delete()
StateType.objects.filter(project=self).delete()
LeafType.objects.filter(project=self).delete()
super().delete(*args, **kwargs)
class Version(Model):
name = CharField(max_length=128)
description = CharField(max_length=1024, blank=True)
number = PositiveIntegerField()
project = ForeignKey(Project, on_delete=CASCADE)
created_datetime = DateTimeField(auto_now_add=True, null=True, blank=True)
created_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='version_created_by')
show_empty = BooleanField(default=True)
""" Tells the UI to show this version even if the current media does not
have any annotations.
"""
bases = ManyToManyField('self', symmetrical=False, blank=True)
""" This version is a patch to an existing version. A use-case here is using one version
for each generation of a state-based inference algorithm; all referencing localizations
in another layer.
"""
def __str__(self):
out = f"{self.name}"
if self.description:
out += f" | {self.description}"
return out
class WithOtherDetails(Model):
other_details = TextField('alte detalii', max_length=500, blank=True, default=None)
vices = TextField('vicii', max_length=500, blank=True, default=None)
display_expiry_date = DateTimeField('dată expirare afişare', blank=True, default=None)
disponibility = TextField('disponibilitate proprietate', blank=True, default=None)
class Meta:
abstract = True
class User(AbstractUser):
objects = TatorUserManager()
cognito_id = UUIDField(primary_key=False,
db_index=True,
null=True,
blank=True,
editable=False)
middle_initial = CharField(max_length=1)
initials = CharField(max_length=3)
last_login = DateTimeField(null=True, blank=True)
last_failed_login = DateTimeField(null=True, blank=True)
failed_login_count = IntegerField(default=0)
def __str__(self):
if self.first_name or self.last_name:
return f"{self.first_name} {self.last_name}"
else:
return "---"
class EntityBase(PolymorphicModel):
project = ForeignKey(Project, on_delete=CASCADE, null=True, blank=True)
meta = ForeignKey(EntityTypeBase, on_delete=CASCADE)
""" Meta points to the defintion of the attribute field. That is
a handful of AttributeTypes are associated to a given EntityType
that is pointed to by this value. That set describes the `attribute`
field of this structure. """
attributes = JSONField(null=True, blank=True)
created_datetime = DateTimeField(auto_now_add=True, null=True, blank=True)
created_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='created_by')
modified_datetime = DateTimeField(auto_now=True, null=True, blank=True)
modified_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='modified_by')
class FaceRecognitionRectangleSubjectDataSuggestion(models.Model):
face_recognition_rectangle = models.ForeignKey(
FaceRecognitionRectangle,
on_delete=CASCADE,
related_name='face_recognition_rectangle')
proposer = models.ForeignKey(Profile,
on_delete=CASCADE,
related_name='subject_data_proposer')
gender = models.PositiveSmallIntegerField(choices=GENDER, null=True)
age = models.PositiveSmallIntegerField(choices=AGE, null=True)
created = DateTimeField(auto_now_add=True, db_index=True)
class LayerMeta(Model):
"""
Immutable state of layer uploading & geoprocessing progress.
To maintain an audit trail of each status change for a layer, these
records should *not* be mutated. Instead, a new record should be created
for each status change.
The data in this table will primarily be maintained by the
geoprocessing side of things.
"""
layer = ForeignKey(Layer, related_name='layer_metas')
state = CharField(max_length=16)
error = TextField(null=True, blank=True)
thumb_small = URLField(
null=True,
blank=True,
help_text='80x80 pixels',
)
thumb_large = URLField(
null=True,
blank=True,
help_text='400x150 pixels',
)
created_at = DateTimeField(auto_now_add=True)
# TileJSON fields
min_zoom = IntegerField(default=0)
max_zoom = IntegerField(default=11)
bounds = CharField(
null=True,
max_length=120,
help_text='JSON array',
)
center = CharField(
null=True,
max_length=60,
help_text='JSON array',
)
def to_json(self):
return {
'id': self.id,
'state': self.state,
'error': self.error,
'thumb_small': self.thumb_small,
'thumb_large': self.thumb_large,
'created_at': self.created_at.isoformat(),
'min_zoom': self.min_zoom,
'max_zoom': self.max_zoom,
'bounds': self.bounds,
'center': self.center,
}
class UserFavoriteLayer(Model):
"""
Created when a layer is "starred" and destroyed when that layer
is "un-starred".
Users may "star" their own layers or published layers.
"""
user = ForeignKey(User)
layer = ForeignKey(Layer, related_name='favorites')
created_at = DateTimeField(auto_now_add=True)
def __unicode__(self):
return '{0} -> {1}'.format(self.user.username, self.layer.name)
class Project(Model):
name = CharField(max_length=128)
creator = ForeignKey(User,
on_delete=PROTECT,
related_name='creator',
db_column='creator')
organization = ForeignKey(Organization,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='organization')
created = DateTimeField(auto_now_add=True)
size = BigIntegerField(default=0)
"""Size of all media in project in bytes.
"""
num_files = IntegerField(default=0)
duration = BigIntegerField(default=0)
""" Duration of all videos in this project.
"""
summary = CharField(max_length=1024)
filter_autocomplete = JSONField(null=True, blank=True)
attribute_type_uuids = JSONField(default=dict, null=True, blank=True)
enable_downloads = BooleanField(default=True)
thumb = CharField(max_length=1024, null=True, blank=True)
usernames = ArrayField(CharField(max_length=256), default=list)
""" Mapping between attribute type names and UUIDs. Used internally for
maintaining elasticsearch field aliases.
"""
def has_user(self, user_id):
return self.membership_set.filter(user_id=user_id).exists()
def user_permission(self, user_id):
permission = None
qs = self.membership_set.filter(user_id=user_id)
if qs.exists():
permission = qs[0].permission
return permission
def __str__(self):
return self.name
def delete(self, *args, **kwargs):
Version.objects.filter(project=self).delete()
MediaType.objects.filter(project=self).delete()
LocalizationType.objects.filter(project=self).delete()
StateType.objects.filter(project=self).delete()
LeafType.objects.filter(project=self).delete()
super().delete(*args, **kwargs)
class Version(Model):
name = CharField(max_length=128)
description = CharField(max_length=1024, blank=True)
number = PositiveIntegerField()
project = ForeignKey(Project, on_delete=CASCADE)
created_datetime = DateTimeField(auto_now_add=True, null=True, blank=True)
created_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='version_created_by')
show_empty = BooleanField(default=False)
""" Tells the UI to show this version even if the current media does not
have any annotations.
"""
def __str__(self):
out = f"{self.name}"
if self.description:
out += f" | {self.description}"
return out
class State(Model):
"""
A State is an event that occurs, potentially independent, from that of
a media element. It is associated with 0 (1 to be useful) or more media
elements. If a frame is supplied it was collected at that time point.
"""
project = ForeignKey(Project,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='project')
meta = ForeignKey(StateType,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='meta')
""" Meta points to the defintion of the attribute field. That is
a handful of AttributeTypes are associated to a given EntityType
that is pointed to by this value. That set describes the `attribute`
field of this structure. """
attributes = JSONField(null=True, blank=True)
""" Values of user defined attributes. """
created_datetime = DateTimeField(auto_now_add=True, null=True, blank=True)
created_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='state_created_by',
db_column='created_by')
modified_datetime = DateTimeField(auto_now=True, null=True, blank=True)
modified_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='state_modified_by',
db_column='modified_by')
version = ForeignKey(Version,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='version')
modified = BooleanField(default=True, null=True, blank=True)
""" Indicates whether an annotation is original or modified.
null: Original upload, no modifications.
false: Original upload, but was modified or deleted.
true: Modified since upload or created via web interface.
"""
media = ManyToManyField(Media, related_name='media')
localizations = ManyToManyField(Localization)
segments = JSONField(null=True, blank=True)
color = CharField(null=True, blank=True, max_length=8)
frame = PositiveIntegerField(null=True, blank=True)
extracted = ForeignKey(Media,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='extracted',
db_column='extracted')
def selectOnMedia(media_id):
return State.objects.filter(media__in=media_id)
class Localization(Model):
project = ForeignKey(Project,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='project')
meta = ForeignKey(LocalizationType,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='meta')
""" Meta points to the defintion of the attribute field. That is
a handful of AttributeTypes are associated to a given LocalizationType
that is pointed to by this value. That set describes the `attribute`
field of this structure. """
attributes = JSONField(null=True, blank=True)
""" Values of user defined attributes. """
created_datetime = DateTimeField(auto_now_add=True, null=True, blank=True)
created_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='localization_created_by',
db_column='created_by')
modified_datetime = DateTimeField(auto_now=True, null=True, blank=True)
modified_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='localization_modified_by',
db_column='modified_by')
user = ForeignKey(User, on_delete=PROTECT, db_column='user')
media = ForeignKey(Media,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='media')
frame = PositiveIntegerField(null=True, blank=True)
thumbnail_image = ForeignKey(Media,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='localization_thumbnail_image',
db_column='thumbnail_image')
version = ForeignKey(Version,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='version')
modified = BooleanField(default=True, null=True, blank=True)
""" Indicates whether an annotation is original or modified.
null: Original upload, no modifications.
false: Original upload, but was modified or deleted.
true: Modified since upload or created via web interface.
"""
x = FloatField(null=True, blank=True)
""" Horizontal position."""
y = FloatField(null=True, blank=True)
""" Vertical position."""
u = FloatField(null=True, blank=True)
""" Horizontal vector component for lines."""
v = FloatField(null=True, blank=True)
""" Vertical vector component for lines. """
width = FloatField(null=True, blank=True)
""" Width for boxes."""
height = FloatField(null=True, blank=True)
""" Height for boxes."""
parent = ForeignKey("self",
on_delete=SET_NULL,
null=True,
blank=True,
db_column='parent')
""" Pointer to localization in which this one was generated from """
class Media(Model):
"""
Fields:
original: Originally uploaded file. Users cannot interact with it except
by downloading it.
.. deprecated :: Use media_files object
segment_info: File for segment files to support MSE playback.
.. deprecated :: Use meda_files instead
media_files: Dictionary to contain a map of all files for this media.
The schema looks like this:
.. code-block ::
map = {"archival": [ VIDEO_DEF, VIDEO_DEF,... ],
"streaming": [ VIDEO_DEF, VIDEO_DEF, ... ],
<"audio": [AUDIO_DEF]>}
video_def = {"path": <path_to_disk>,
"codec": <human readable codec>,
"resolution": [<vertical pixel count, e.g. 720>, width]
audio_def = {"path": <path_to_disk>,
"codec": <human readable codec>}
###################
# Optional Fields #
###################
# Path to the segments.json file for streaming files.
# not expected/required for archival. Required for
# MSE playback with seek support for streaming files.
segment_info = <path_to_json>
# If supplied will use this instead of currently
# connected host. e.g. https://example.com
"host": <host url>
# If specified will be used for HTTP authorization
# in the request for media. I.e. "bearer <token>"
"http_auth": <http auth header>
# Example mime: 'video/mp4; codecs="avc1.64001e"'
# Only relevant for straming files, will assume
# example above if not present.
"codec_mime": <mime for MSE decode>
"codec_description": <description other than codec>}
"""
project = ForeignKey(Project,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='project')
meta = ForeignKey(MediaType,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='meta')
""" Meta points to the defintion of the attribute field. That is
a handful of AttributeTypes are associated to a given MediaType
that is pointed to by this value. That set describes the `attribute`
field of this structure. """
attributes = JSONField(null=True, blank=True)
""" Values of user defined attributes. """
gid = CharField(max_length=36, null=True, blank=True)
""" Group ID for the upload that created this media. Note we intentionally do
not use UUIDField because this field is provided by the uploader and not
guaranteed to be an actual UUID. """
uid = CharField(max_length=36, null=True, blank=True)
""" Unique ID for the upload that created this media. Note we intentionally do
not use UUIDField because this field is provided by the uploader and not
guaranteed to be an actual UUID. """
created_datetime = DateTimeField(auto_now_add=True, null=True, blank=True)
created_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='media_created_by',
db_column='created_by')
modified_datetime = DateTimeField(auto_now=True, null=True, blank=True)
modified_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='media_modified_by',
db_column='modified_by')
name = CharField(max_length=256)
md5 = SlugField(max_length=32)
""" md5 hash of the originally uploaded file. """
file = FileField(null=True, blank=True)
last_edit_start = DateTimeField(null=True, blank=True)
""" Start datetime of a session in which the media's annotations were edited.
"""
last_edit_end = DateTimeField(null=True, blank=True)
""" End datetime of a session in which the media's annotations were edited.
"""
original = FilePathField(path=settings.RAW_ROOT, null=True, blank=True)
thumbnail = ImageField(null=True, blank=True)
thumbnail_gif = ImageField(null=True, blank=True)
num_frames = IntegerField(null=True, blank=True)
fps = FloatField(null=True, blank=True)
codec = CharField(null=True, blank=True, max_length=256)
width = IntegerField(null=True)
height = IntegerField(null=True)
segment_info = FilePathField(path=settings.MEDIA_ROOT,
null=True,
blank=True)
media_files = JSONField(null=True, blank=True)
def update_media_files(self, media_files):
""" Updates media files by merging a new key into existing JSON object.
"""
# Handle null existing value.
if self.media_files is None:
self.media_files = {}
# Append to existing definitions.
new_streaming = media_files.get('streaming', [])
old_streaming = self.media_files.get('streaming', [])
streaming = new_streaming + old_streaming
new_archival = media_files.get('archival', [])
old_archival = self.media_files.get('archival', [])
archival = new_archival + old_archival
new_audio = media_files.get('audio', [])
old_audio = self.media_files.get('audio', [])
audio = new_audio + old_audio
for fp in new_streaming:
path = fp['path']
seg_path = fp['segment_info']
Resource.add_resource(path)
Resource.add_resource(seg_path)
for fp in new_archival:
Resource.add_resource(fp['path'])
for fp in new_audio:
Resource.add_resource(fp['path'])
# Only fill in a key if it has at least one definition.
self.media_files = {}
if streaming:
streaming.sort(key=lambda x: x['resolution'][0], reverse=True)
self.media_files['streaming'] = streaming
if archival:
self.media_files['archival'] = archival
if audio:
self.media_files['audio'] = audio
# Handle roi, layout, and quality
for x in ['layout', 'ids', 'quality']:
if x in media_files:
self.media_files[x] = media_files[x]
class Media(Model):
"""
Fields:
original: Originally uploaded file. Users cannot interact with it except
by downloading it.
.. deprecated :: Use media_files object
segment_info: File for segment files to support MSE playback.
.. deprecated :: Use meda_files instead
media_files: Dictionary to contain a map of all files for this media.
The schema looks like this:
.. code-block ::
map = {"archival": [ VIDEO_DEF, VIDEO_DEF,... ],
"streaming": [ VIDEO_DEF, VIDEO_DEF, ... ],
<"audio": [AUDIO_DEF]>}
video_def = {"path": <path_to_disk>,
"codec": <human readable codec>,
"resolution": [<vertical pixel count, e.g. 720>, width]
audio_def = {"path": <path_to_disk>,
"codec": <human readable codec>}
###################
# Optional Fields #
###################
# Path to the segments.json file for streaming files.
# not expected/required for archival. Required for
# MSE playback with seek support for streaming files.
segment_info = <path_to_json>
# If supplied will use this instead of currently
# connected host. e.g. https://example.com
"host": <host url>
# If specified will be used for HTTP authorization
# in the request for media. I.e. "bearer <token>"
"http_auth": <http auth header>
# Example mime: 'video/mp4; codecs="avc1.64001e"'
# Only relevant for straming files, will assume
# example above if not present.
"codec_mime": <mime for MSE decode>
"codec_description": <description other than codec>}
"""
project = ForeignKey(Project,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='project',
related_name='media_project')
meta = ForeignKey(MediaType,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='meta')
""" Meta points to the defintion of the attribute field. That is
a handful of AttributeTypes are associated to a given MediaType
that is pointed to by this value. That set describes the `attribute`
field of this structure. """
attributes = JSONField(null=True, blank=True)
""" Values of user defined attributes. """
gid = CharField(max_length=36, null=True, blank=True)
""" Group ID for the upload that created this media. Note we intentionally do
not use UUIDField because this field is provided by the uploader and not
guaranteed to be an actual UUID. """
uid = CharField(max_length=36, null=True, blank=True)
""" Unique ID for the upload that created this media. Note we intentionally do
not use UUIDField because this field is provided by the uploader and not
guaranteed to be an actual UUID. """
created_datetime = DateTimeField(auto_now_add=True, null=True, blank=True)
created_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='media_created_by',
db_column='created_by')
modified_datetime = DateTimeField(auto_now=True, null=True, blank=True)
modified_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='media_modified_by',
db_column='modified_by')
name = CharField(max_length=256)
md5 = SlugField(max_length=32)
""" md5 hash of the originally uploaded file. """
file = FileField(null=True, blank=True)
last_edit_start = DateTimeField(null=True, blank=True)
""" Start datetime of a session in which the media's annotations were edited.
"""
last_edit_end = DateTimeField(null=True, blank=True)
""" End datetime of a session in which the media's annotations were edited.
"""
original = FilePathField(path=settings.RAW_ROOT, null=True, blank=True)
thumbnail = ImageField(null=True, blank=True)
thumbnail_gif = ImageField(null=True, blank=True)
num_frames = IntegerField(null=True, blank=True)
fps = FloatField(null=True, blank=True)
codec = CharField(null=True, blank=True, max_length=256)
width = IntegerField(null=True)
height = IntegerField(null=True)
segment_info = FilePathField(path=settings.MEDIA_ROOT,
null=True,
blank=True)
media_files = JSONField(null=True, blank=True)
recycled_from = ForeignKey(Project,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='recycled_from')
class Leaf(Model):
project = ForeignKey(Project,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='project')
meta = ForeignKey(LeafType,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='meta')
""" Meta points to the defintion of the attribute field. That is
a handful of AttributeTypes are associated to a given EntityType
that is pointed to by this value. That set describes the `attribute`
field of this structure. """
attributes = JSONField(null=True, blank=True)
""" Values of user defined attributes. """
created_datetime = DateTimeField(auto_now_add=True, null=True, blank=True)
created_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='leaf_created_by',
db_column='created_by')
modified_datetime = DateTimeField(auto_now=True, null=True, blank=True)
modified_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='leaf_modified_by',
db_column='modified_by')
parent = ForeignKey('self',
on_delete=SET_NULL,
blank=True,
null=True,
db_column='parent')
path = PathField(unique=True)
name = CharField(max_length=255)
class Meta:
verbose_name_plural = "Leaves"
def __str__(self):
return str(self.path)
def depth(self):
return Leaf.objects.annotate(depth=Depth('path')).get(pk=self.pk).depth
def subcategories(self, minLevel=1):
return Leaf.objects.select_related('parent').filter(
path__descendants=self.path,
path__depth__gte=self.depth() + minLevel)
def computePath(self):
""" Returns the string representing the path element """
pathStr = self.name.replace(" ", "_").replace("-", "_").replace(
"(", "_").replace(")", "_")
if self.parent:
pathStr = self.parent.computePath() + "." + pathStr
elif self.project:
projName = self.project.name.replace(" ", "_").replace(
"-", "_").replace("(", "_").replace(")", "_")
pathStr = projName + "." + pathStr
return pathStr
class Layer(Model):
"""
Represents a single Image Layer which may contain one or more
geospatial images.
"""
class Meta:
unique_together = ('user', 'name')
user = ForeignKey(User)
name = CharField(max_length=255)
slug = SlugField(max_length=255, blank=True)
status_created = DateTimeField(auto_now_add=True)
status_validate_start = DateTimeField(null=True, blank=True)
status_validate_end = DateTimeField(null=True, blank=True)
status_thumbnail_start = DateTimeField(null=True, blank=True)
status_thumbnail_end = DateTimeField(null=True, blank=True)
status_create_cluster_start = DateTimeField(null=True, blank=True)
status_create_cluster_end = DateTimeField(null=True, blank=True)
status_chunk_start = DateTimeField(null=True, blank=True)
status_chunk_end = DateTimeField(null=True, blank=True)
status_mosaic_start = DateTimeField(null=True, blank=True)
status_mosaic_end = DateTimeField(null=True, blank=True)
status_failed = DateTimeField(null=True, blank=True)
status_completed = DateTimeField(null=True, blank=True)
status_heartbeat = DateTimeField(null=True, blank=True)
status_validate_error = CharField(max_length=255, blank=True, null=True)
status_thumbnail_error = CharField(max_length=255, blank=True, null=True)
status_create_cluster_error = CharField(max_length=255,
blank=True,
null=True)
status_chunk_error = CharField(max_length=255, blank=True, null=True)
status_mosaic_error = CharField(max_length=255, blank=True, null=True)
status_failed_error = CharField(max_length=255, blank=True, null=True)
description = TextField(blank=True)
organization = CharField(max_length=255, blank=True, null=True)
is_public = BooleanField(default=False)
capture_start = DateField(blank=True, null=True)
capture_end = DateField(blank=True, null=True)
min_x = FloatField(default=None, blank=True, null=True)
max_x = FloatField(default=None, blank=True, null=True)
min_y = FloatField(default=None, blank=True, null=True)
max_y = FloatField(default=None, blank=True, null=True)
area = FloatField(default=0, blank=True, null=True)
area_unit = CharField(
blank=True,
max_length=18,
choices=enums.AREA_UNIT_CHOICES,
default=enums.SQ_KM,
)
projection = CharField(
blank=True,
max_length=18,
choices=enums.PROJECTION_CHOICES,
default=enums.WGS84,
help_text='Source Projection',
)
srid = CharField(
blank=True,
max_length=18,
choices=enums.SRID_CHOICES,
default=enums.WGS84,
help_text='Source SRS',
)
tile_srid = CharField(blank=True,
max_length=18,
choices=enums.SRID_CHOICES,
default=enums.WGS84,
help_text='Tile SRS')
tile_format = CharField(
blank=True,
max_length=18,
choices=enums.TILE_FORMAT_CHOICES,
default=enums.OVER_PNG32,
)
tile_origin = CharField(
blank=True,
max_length=18,
choices=enums.TILE_ORIGIN_CHOICES,
default=enums.TOPLEFT,
help_text='Tiling Scheme',
)
resampling = CharField(
blank=True,
max_length=18,
choices=enums.TILE_RESAMPLING_CHOICES,
default=enums.BILINEAR,
)
transparency = CharField(
blank=True,
max_length=18,
help_text='Hexadecimal (Ex. #00FF00)',
)
dismissed = BooleanField(blank=True, default=False)
created_at = DateTimeField(auto_now_add=True)
updated_at = DateTimeField(auto_now=True)
deleted_at = DateTimeField(null=True, blank=True)
status_updated_at = DateTimeField(default=datetime.now)
thumb_small_key = CharField(max_length=255,
blank=True,
default='',
help_text='S3 key for small thumbnail')
thumb_large_key = CharField(max_length=255,
blank=True,
default='',
help_text='S3 key for large thumbnail')
def has_copy_images(self):
for image in self.layer_images.all():
if image.is_copy_image():
return True
return False
def save(self, *args, **kwargs):
self.slug = slugify(self.name)
super(Layer, self).save(*args, **kwargs)
def to_json(self):
"""
Return JSON serializable model data.
Note: Prefetch all related foreign key relationships before
calling this method for optimal performance.
"""
tags = [m.to_json() for m in self.layer_tags.all()]
images = [m.to_json() for m in self.layer_images.all()]
capture_start = self.capture_start.isoformat() \
if self.capture_start else None
capture_end = self.capture_end.isoformat() \
if self.capture_end else None
return {
'id':
self.id,
'name':
self.name,
'slug':
self.slug,
'description':
self.description,
'organization':
self.organization,
'is_public':
self.is_public,
'capture_start':
capture_start,
'capture_end':
capture_end,
'area':
self.area,
'area_unit':
self.area_unit,
'bounds':
self.get_bounds(),
'projection':
self.projection,
'srid':
self.srid,
'tile_srid':
self.tile_srid,
'tile_format':
self.tile_format,
'tile_origin':
self.tile_origin,
'resampling':
self.resampling,
'transparency':
self.transparency,
'status_created':
self.status_created is not None,
'status_validate_start':
self.status_validate_start is not None,
'status_validate_end':
self.status_validate_end is not None,
'status_thumbnail_start':
self.status_thumbnail_start is not None,
'status_thumbnail_end':
self.status_thumbnail_end is not None,
'status_create_cluster_start': (self.status_create_cluster_start
is not None),
'status_create_cluster_end': (self.status_create_cluster_end
is not None),
'status_chunk_start':
self.status_chunk_start is not None,
'status_chunk_end':
self.status_chunk_end is not None,
'status_mosaic_start':
self.status_mosaic_start is not None,
'status_mosaic_end':
self.status_mosaic_end is not None,
'status_failed':
self.status_failed is not None,
'status_completed':
self.status_completed is not None,
'status_heartbeat':
self.status_completed is not None,
'status_validate_error':
self.status_validate_error,
'status_thumbnail_error':
self.status_thumbnail_error,
'status_create_cluster_error':
self.status_create_cluster_error,
'status_chunk_error':
self.status_chunk_error,
'status_mosaic_error':
self.status_mosaic_error,
'status_failed_error':
self.status_failed_error,
'created_at':
self.created_at.isoformat(),
'updated_at':
self.updated_at.isoformat(),
'status_updated_at':
self.created_at.isoformat(),
'thumb_small':
generate_thumb_url(self.thumb_small_key),
'thumb_large':
generate_thumb_url(self.thumb_large_key),
# Foreign key fields
'tags':
tags,
'images':
images,
'username':
self.user.username,
# Generated fields
'url':
self.get_absolute_url(),
'meta_url':
self.get_meta_url(),
'favorite_url':
self.get_favorite_url(),
'dismiss_url':
self.get_dismiss_url(),
'retry_url':
self.get_retry_url(),
'tile_url':
self.get_tile_url(),
}
def get_bounds(self):
if all([self.min_x, self.min_y, self.max_x, self.max_y]):
return [
[self.min_y, self.min_x],
[self.max_y, self.max_x],
]
return None
def retry_possible(self):
"""
Returns true if it is possible to retry processing a layer.
"""
return self.status_failed is not None
def reset(self):
"""
Resets fields to prepare for a retry.
"""
self.status_validate_start = None
self.status_validate_end = None
self.status_thumbnail_start = None
self.status_thumbnail_end = None
self.status_create_cluster_start = None
self.status_create_cluster_end = None
self.status_chunk_start = None
self.status_chunk_end = None
self.status_mosaic_start = None
self.status_mosaic_end = None
self.status_failed = None
self.status_completed = None
self.status_heartbeat = None
self.status_validate_error = None
self.status_thumbnail_error = None
self.status_create_cluster_error = None
self.status_chunk_error = None
self.status_mosaic_error = None
self.status_failed_error = None
self.save()
for image in self.layer_images.all():
image.reset()
def get_absolute_url(self):
kwargs = {
'layer_id': self.id,
'username': self.user.username,
}
return reverse('layer_detail', kwargs=kwargs)
def get_meta_url(self):
kwargs = {
'layer_id': self.id,
'username': self.user.username,
}
return reverse('layer_meta', kwargs=kwargs)
def get_favorite_url(self):
kwargs = {
'layer_id': self.id,
}
return reverse('create_or_destroy_favorite', kwargs=kwargs)
def get_retry_url(self):
return reverse('layer_retry')
def get_dismiss_url(self):
return reverse('layer_dismiss')
def get_tile_url(self):
url = 'https://s3.amazonaws.com/%s/%d/{z}/{x}/{y}.png'
return url % (settings.AWS_TILES_BUCKET, self.id)
def get_tile_bucket_path(self):
return 's3://{}/{}'.format(settings.AWS_TILES_BUCKET, self.id)
def process_failed_heartbeat(self):
self.status_heartbeat = datetime.now()
self.save()
def update_status_start(self, status):
value = datetime.now()
if status == enums.STATUS_VALIDATE:
self.status_validate_start = (value
if self.status_validate_start is None
else self.status_validate_start)
elif status == enums.STATUS_THUMBNAIL:
self.status_thumbnail_start = (value if
self.status_thumbnail_start is None
else self.status_thumbnail_start)
elif status == enums.STATUS_CREATE_CLUSTER:
self.status_create_cluster_start = \
(value if self.status_create_cluster_start is None else
self.status_create_cluster_start)
elif status == enums.STATUS_CHUNK:
self.status_chunk_start = (value if self.status_chunk_start is None
else self.status_chunk_start)
elif status == enums.STATUS_MOSAIC:
self.status_mosaic_start = (value
if self.status_mosaic_start is None
else self.status_mosaic_start)
def update_status_end(self, status, error_message=None):
value = datetime.now()
if status == enums.STATUS_VALIDATE:
self.status_validate_end = value
self.status_validate_error = error_message
elif status == enums.STATUS_THUMBNAIL:
self.status_thumbnail_end = value
self.status_thumbnail_error = error_message
elif status == enums.STATUS_CREATE_CLUSTER:
self.status_create_cluster_end = value
self.status_create_cluster_error = error_message
elif status == enums.STATUS_CHUNK:
self.status_chunk_end = value
self.status_chunk_error = error_message
elif status == enums.STATUS_MOSAIC:
self.status_mosaic_end = value
self.status_mosaic_error = error_message
elif status == enums.STATUS_COMPLETED:
if error_message is not None:
raise StatusMismatchError(
'Completed status does not accept errors.')
self.status_completed = value
# To be safe, unset the failed status and error.
self.status_failed = None
self.status_failed_error = None
elif status == enums.STATUS_FAILED:
if self.status_completed is not None:
raise StatusMismatchError(
'Cannot mark completed layer as failed.')
self.status_failed_error = error_message
# If we had any error message mark the generic failed field.
if error_message is not None:
if self.status_completed is not None:
raise StatusMismatchError(
'Cannot set errors on completed layer.')
self.status_failed = value
def mark_failed(self):
value = datetime.now()
self.status_failed = value
def set_bounds(self, bounds):
self.min_x = bounds[0]
self.max_x = bounds[1]
self.min_y = bounds[2]
self.max_y = bounds[3]
self.save(update_fields=['min_x', 'max_x', 'min_y', 'max_y'])
def __unicode__(self):
return '{0} -> {1}'.format(self.user.username, self.name)
class LayerImage(Model):
"""
Geospatial image uploaded by the user.
`meta_json` may be populated during geoprocessing and should contain
a serialized JSON blob of image metadata.
This blob should be in the form of key value pairs (object literal)
and will be displayed as-is.
"""
layer = ForeignKey(Layer, related_name='layer_images')
priority = IntegerField(
default=0,
help_text='The order which images are layered (starting from 0)')
thumb_small_key = CharField(max_length=255,
blank=True,
default='',
help_text='S3 key for small thumbnail')
thumb_large_key = CharField(max_length=255,
blank=True,
default='',
help_text='S3 key for large thumbnail')
meta_json = TextField(
null=True,
blank=True,
help_text='Serialized JSON of image metadata',
)
min_x = FloatField(default=None, blank=True, null=True)
max_x = FloatField(default=None, blank=True, null=True)
min_y = FloatField(default=None, blank=True, null=True)
max_y = FloatField(default=None, blank=True, null=True)
file_name = CharField(
blank=False,
default='',
max_length=255,
help_text='Filename of original file',
)
s3_uuid = UUIDField(default=uuid.uuid4, editable=False)
file_extension = CharField(blank=False,
default='',
max_length=10,
help_text='Extension of file')
bucket_name = CharField(blank=False,
default='',
max_length=255,
help_text='Name of S3 bucket')
source_s3_bucket_key = CharField(
blank=True,
null=True,
max_length=255,
help_text='S3 <bucket>/<key> for source image (optional)')
status_created = DateTimeField(auto_now_add=True)
status_transfer_start = DateTimeField(null=True, blank=True)
status_transfer_end = DateTimeField(null=True, blank=True)
status_validate_start = DateTimeField(null=True, blank=True)
status_validate_end = DateTimeField(null=True, blank=True)
status_thumbnail_start = DateTimeField(null=True, blank=True)
status_thumbnail_end = DateTimeField(null=True, blank=True)
status_upload_error = CharField(max_length=255, blank=True, null=True)
status_transfer_error = CharField(max_length=255, blank=True, null=True)
status_validate_error = CharField(max_length=255, blank=True, null=True)
status_thumbnail_error = CharField(max_length=255, blank=True, null=True)
def reset(self):
"""
Resets fields to prepare for a retry.
"""
self.status_transfer_start = None
self.status_transfer_end = None
self.status_validate_start = None
self.status_validate_end = None
self.status_thumbnail_start = None
self.status_thumbnail_end = None
self.status_upload_error = None
self.status_transfer_error = None
self.status_validate_error = None
self.status_thumbnail_error = None
self.save()
def has_been_validated(self):
return self.status_validate_end is not None
def is_copy_image(self):
return self.source_s3_bucket_key is not None
def get_s3_key(self):
return '%d-%s.%s' % (self.layer.user.id, self.s3_uuid,
self.file_extension)
def get_s3_uri(self):
return 's3://{}/{}-{}.{}'.format(self.bucket_name, self.layer.user_id,
self.s3_uuid, self.file_extension)
def to_json(self):
return {
'id': self.id,
'thumb_small': generate_thumb_url(self.thumb_small_key),
'thumb_large': generate_thumb_url(self.thumb_large_key),
'meta_json': self.meta_json,
'min_x': self.min_x,
'max_x': self.max_x,
'min_y': self.min_y,
'max_y': self.max_y,
'file_name': self.file_name,
's3_uuid': str(self.s3_uuid),
'file_extension': self.file_extension,
'bucket_name': self.bucket_name,
'source_s3_bucket_key': self.source_s3_bucket_key,
'status_created': self.status_created is not None,
'status_transfer_start': self.status_transfer_start is not None,
'status_transfer_end': self.status_transfer_end is not None,
'status_validate_start': self.status_validate_start is not None,
'status_validate_end': self.status_validate_end is not None,
'status_thumbnail_start': self.status_thumbnail_start is not None,
'status_thumbnail_end': self.status_thumbnail_end is not None,
'status_upload_error': self.status_upload_error,
'status_transfer_error': self.status_transfer_error,
'status_validate_error': self.status_validate_error,
'status_thumbnail_error': self.status_thumbnail_error,
'source_s3_bucket_key': self.source_s3_bucket_key
}
def update_status_start(self, status):
value = datetime.now()
if status == enums.STATUS_TRANSFER:
self.status_transfer_start = (value
if self.status_transfer_start is None
else self.status_transfer_start)
elif status == enums.STATUS_VALIDATE:
self.status_validate_start = (value
if self.status_validate_start is None
else self.status_validate_start)
elif status == enums.STATUS_THUMBNAIL:
self.status_thumbnail_start = (value if
self.status_thumbnail_start is None
else self.status_thumbnail_start)
def update_status_end(self, status, error_message=None):
value = datetime.now()
if status == enums.STATUS_TRANSFER:
self.status_transfer_end = value
self.status_transfer_error = error_message
elif status == enums.STATUS_VALIDATE:
self.status_validate_end = value
self.status_validate_error = error_message
elif status == enums.STATUS_THUMBNAIL:
self.status_thumbnail_end = value
self.status_thumbnail_error = error_message
def set_bounds(self, bounds):
self.min_x = bounds[0]
self.max_x = bounds[1]
self.min_y = bounds[2]
self.max_y = bounds[3]
self.save(update_fields=['min_x', 'max_x', 'min_y', 'max_y'])