class TreeLeaf(EntityBase):
parent=ForeignKey('self', on_delete=SET_NULL, blank=True, null=True)
path=PathField(unique=True)
name = CharField(max_length=255)
class Meta:
verbose_name_plural = "TreeLeaves"
def __str__(self):
return str(self.path)
def depth(self):
return TreeLeaf.objects.annotate(depth=Depth('path')).get(pk=self.pk).depth
def subcategories(self, minLevel=1):
return TreeLeaf.objects.select_related('parent').filter(
path__descendants=self.path,
path__depth__gte=self.depth()+minLevel
)
def computePath(self):
""" Returns the string representing the path element """
pathStr=self.name.replace(" ","_").replace("-","_").replace("(","_").replace(")","_")
if self.parent:
pathStr=self.parent.computePath()+"."+pathStr
elif self.project:
projName=self.project.name.replace(" ","_").replace("-","_").replace("(","_").replace(")","_")
pathStr=projName+"."+pathStr
return pathStr
class Category(models.Model):
name = models.CharField(max_length=140)
path = PathField(unique=True)
class Meta:
verbose_name_plural = "categories"
def __str__(self):
return self.path
def subcategories(self):
return Category.objects.filter(path__descendant=self.path,
path__nlevel=NLevel(self.path) + 1)
def test_registered_lookups():
registered_lookups = PathField.get_lookups()
assert "ancestors" in registered_lookups, "Missing 'ancestors' in lookups"
assert registered_lookups["ancestors"] is lookups.AncestorLookup
assert "descendants" in registered_lookups, "Missing 'descendants' in lookups"
assert registered_lookups["descendants"] is lookups.DescendantLookup
assert "match" in registered_lookups, "Missing 'match' in lookups"
assert registered_lookups["match"] is lookups.MatchLookup
assert "depth" in registered_lookups, "Missing 'depth' in lookups"
assert registered_lookups["depth"] is functions.NLevel
assert "contains" in registered_lookups, "Missing 'contains' in lookups"
assert registered_lookups["contains"] is lookups.ContainsLookup
class Contest(models.Model):
resource = models.ForeignKey(Resource, on_delete=models.CASCADE)
title = models.CharField(max_length=2048)
slug = models.CharField(max_length=2048, null=True, blank=True, db_index=True)
title_path = PathField(null=True, blank=True, db_index=True)
start_time = models.DateTimeField()
end_time = models.DateTimeField()
duration_in_secs = models.IntegerField(null=False, blank=True)
url = models.CharField(max_length=255)
key = models.CharField(max_length=255)
host = models.CharField(max_length=255)
uid = models.CharField(max_length=100, null=True, blank=True)
edit = models.CharField(max_length=100, null=True, blank=True)
invisible = models.BooleanField(default=False)
standings_url = models.CharField(max_length=2048, null=True, blank=True)
calculate_time = models.BooleanField(default=False)
info = JSONField(default=dict, blank=True)
writers = models.ManyToManyField('ranking.Account', blank=True, related_name='writer_set')
created = models.DateTimeField(auto_now_add=True)
modified = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
was_auto_added = models.BooleanField(default=False)
objects = BaseManager()
visible = VisibleContestManager()
class Meta:
unique_together = ('resource', 'key', )
indexes = [
models.Index(fields=['start_time']),
models.Index(fields=['end_time']),
]
def save(self, *args, **kwargs):
if self.duration_in_secs is None:
self.duration_in_secs = (self.end_time - self.start_time).total_seconds()
self.slug = slug(self.title).strip('-')
self.title_path = self.slug.replace('-', '.')
return super(Contest, self).save(*args, **kwargs)
def is_over(self):
return self.end_time <= timezone.now()
def is_running(self):
return not self.is_over() and self.start_time < timezone.now()
def is_coming(self):
return timezone.now() < self.start_time
@property
def next_time(self):
if self.is_over():
return 0
return int(round(
((
self.end_time
if self.is_running()
else self.start_time
) - timezone.now()).total_seconds()
))
def __str__(self):
return "%s [%d]" % (self.title, self.id)
@property
def duration(self):
return timedelta(seconds=self.duration_in_secs)
# Fix for virtual contest
# return self.end_time - self.start_time
@property
def hr_duration(self):
duration = self.duration
if duration > timedelta(days=999):
return "%d years" % (duration.days // 364)
elif duration > timedelta(days=3):
return "%d days" % duration.days
else:
total = duration.total_seconds()
return "%02d:%02d" % ((total + 1e-9) // 3600, (total + 1e-9) % 3600 // 60)
@classmethod
def month_regex(cls):
if not hasattr(cls, '_month_regex'):
months = itertools.chain(calendar.month_name, calendar.month_abbr)
regex = '|'.join([f'[{m[0]}{m[0].lower()}]{m[1:]}' for m in months if m])
cls._month_regex = rf'\b(?:{regex})\b'
return cls._month_regex
@staticmethod
def title_neighbors_(title, deep, viewed):
viewed.add(title)
if deep == 0:
return
for match in re.finditer(rf'([0-9]+|[A-Z]\b|{Contest.month_regex()})', title):
for delta in (-1, 1):
base_title = title
value = match.group(0)
values = []
if value.isdigit():
value = str(int(value) + delta)
elif len(value) == 1:
value = chr(ord(value) + delta)
else:
mformat = '%b' if len(value) == 3 else '%B'
index = datetime.strptime(value.title(), mformat).month
mformats = ['%b', '%B'] if index == 5 else [mformat]
if not (1 <= index + delta <= 12):
ym = re.search(r'\b[0-9]{4}\b', base_title)
if ym:
year = str(int(ym.group()) + delta)
base_title = base_title[:ym.start()] + year + base_title[ym.end():]
index = (index - 1 + delta) % 12 + 1
for mformat in mformats:
values.append(datetime.strptime(str(index), '%m').strftime(mformat))
values = values or [value]
for value in values:
new_title = base_title[:match.start()] + value + base_title[match.end():]
if new_title in viewed:
continue
Contest.title_neighbors_(new_title, deep=deep - 1, viewed=viewed)
def neighbors(self):
viewed = set()
Contest.title_neighbors_(self.title, deep=1, viewed=viewed)
cond = Q()
for title in viewed:
cond |= Q(title=title)
resource_contests = Contest.objects.filter(resource=self.resource_id)
resource_contests = resource_contests.annotate(has_statistics=Exists('statistics')).filter(has_statistics=True)
for query, order in (
(Q(end_time__lt=self.start_time), '-end_time'),
(Q(start_time__gt=self.end_time), 'start_time'),
):
c = resource_contests.filter(query).order_by(order).first()
if c:
cond |= Q(pk=c.pk)
if self.title_path is not None:
qs = resource_contests.filter(query).exclude(title=self.title)
qs = qs.extra(select={'lcp': f'''nlevel(lca(title_path, '{self.title_path}'))'''})
qs = qs.order_by('-lcp', order)
c = qs.first()
if c and c.lcp:
cond |= Q(pk=c.pk)
qs = resource_contests.filter(cond).exclude(pk=self.pk).order_by('end_time')
return qs
class Leaf(Model):
project = ForeignKey(Project,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='project')
meta = ForeignKey(LeafType,
on_delete=SET_NULL,
null=True,
blank=True,
db_column='meta')
""" Meta points to the defintion of the attribute field. That is
a handful of AttributeTypes are associated to a given EntityType
that is pointed to by this value. That set describes the `attribute`
field of this structure. """
attributes = JSONField(null=True, blank=True)
""" Values of user defined attributes. """
created_datetime = DateTimeField(auto_now_add=True, null=True, blank=True)
created_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='leaf_created_by',
db_column='created_by')
modified_datetime = DateTimeField(auto_now=True, null=True, blank=True)
modified_by = ForeignKey(User,
on_delete=SET_NULL,
null=True,
blank=True,
related_name='leaf_modified_by',
db_column='modified_by')
parent = ForeignKey('self',
on_delete=SET_NULL,
blank=True,
null=True,
db_column='parent')
path = PathField(unique=True)
name = CharField(max_length=255)
class Meta:
verbose_name_plural = "Leaves"
def __str__(self):
return str(self.path)
def depth(self):
return Leaf.objects.annotate(depth=Depth('path')).get(pk=self.pk).depth
def subcategories(self, minLevel=1):
return Leaf.objects.select_related('parent').filter(
path__descendants=self.path,
path__depth__gte=self.depth() + minLevel)
def computePath(self):
""" Returns the string representing the path element """
pathStr = self.name.replace(" ", "_").replace("-", "_").replace(
"(", "_").replace(")", "_")
if self.parent:
pathStr = self.parent.computePath() + "." + pathStr
elif self.project:
projName = self.project.name.replace(" ", "_").replace(
"-", "_").replace("(", "_").replace(")", "_")
pathStr = projName + "." + pathStr
return pathStr
import uuid
# Load the main.view logger
logger = logging.getLogger(__name__)
class Depth(Transform):
lookup_name = "depth"
function = "nlevel"
@property
def output_field(self):
return IntegerField()
PathField.register_lookup(Depth)
FileFormat = [('mp4', 'mp4'), ('webm', 'webm'), ('mov', 'mov')]
ImageFileFormat = [('jpg', 'jpg'), ('png', 'png'), ('bmp', 'bmp'),
('raw', 'raw')]
## Describes different association models in the database
AssociationTypes = [
('Media', 'Relates to one or more media items'),
('Frame', 'Relates to a specific frame in a video'
), #Relates to one or more frames in a video
('Localization', 'Relates to localization(s)')
] #Relates to one-to-many localizations
class MediaAccess(Enum):
class OrgUnit(models.Model):
VALIDATION_NEW = "NEW"
VALIDATION_VALID = "VALID"
VALIDATION_REJECTED = "REJECTED"
VALIDATION_STATUS_CHOICES = (
(VALIDATION_NEW, _("new")),
(VALIDATION_VALID, _("valid")),
(VALIDATION_REJECTED, _("rejected")),
)
name = models.CharField(max_length=255)
uuid = models.TextField(null=True, blank=True, db_index=True)
custom = models.BooleanField(default=False)
validated = models.BooleanField(
default=True, db_index=True) # TO DO : remove in a later migration
validation_status = models.CharField(max_length=25,
choices=VALIDATION_STATUS_CHOICES,
default=VALIDATION_NEW)
version = models.ForeignKey("SourceVersion",
null=True,
blank=True,
on_delete=models.CASCADE)
parent = models.ForeignKey("OrgUnit",
on_delete=models.CASCADE,
null=True,
blank=True)
path = PathField(null=True, blank=True, unique=True)
aliases = ArrayField(CITextField(max_length=255, blank=True),
size=100,
null=True,
blank=True)
org_unit_type = models.ForeignKey(OrgUnitType,
on_delete=models.CASCADE,
null=True,
blank=True)
sub_source = models.TextField(
null=True,
blank=True) # sometimes, in a given source, there are sub sources
source_ref = models.TextField(null=True, blank=True, db_index=True)
geom = MultiPolygonField(null=True, blank=True, srid=4326, geography=True)
simplified_geom = MultiPolygonField(null=True,
blank=True,
srid=4326,
geography=True)
catchment = MultiPolygonField(null=True,
blank=True,
srid=4326,
geography=True)
geom_ref = models.IntegerField(null=True, blank=True)
gps_source = models.TextField(null=True, blank=True)
location = PointField(null=True,
blank=True,
geography=True,
dim=3,
srid=4326)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
creator = models.ForeignKey(User,
null=True,
blank=True,
on_delete=models.SET_NULL)
objects = OrgUnitManager.from_queryset(OrgUnitQuerySet)()
class Meta:
indexes = [GistIndex(fields=["path"], buffering=True)]
def save(self, *args, skip_calculate_path: bool = False, **kwargs):
"""Override default save() to make sure that the path property is calculated and saved,
for this org unit and its children.
:param skip_calculate_path: use with caution - can be useful in scripts where the extra transactions
would be a burden, but the path needs to be set afterwards
"""
if skip_calculate_path:
super().save(*args, **kwargs)
else:
with transaction.atomic():
super().save(*args, **kwargs)
OrgUnit.objects.bulk_update(self.calculate_paths(), ["path"])
def calculate_paths(self,
force_recalculate: bool = False
) -> typing.List["OrgUnit"]:
"""Calculate the path for this org unit and all its children.
This method will check if this org unit path should change. If it is the case (or if force_recalculate is
True), it will update the path property for the org unit and its children, and return all the modified
records.
Please note that this method does not save the modified records. Instead, they are updated in bulk in the
save() method.
:param force_recalculate: calculate path for all descendants, even if this org unit path does not change
"""
# For now, we will skip org units that have a parent without a path.
# The idea is that a management command (set_org_unit_path) will handle the initial seeding of the
# path field, starting at the top of the pyramid. Once this script has been run and the field is filled for
# all org units, this should not happen anymore.
# TODO: remove condition below
if self.parent is not None and self.parent.path is None:
return []
# keep track of updated records
updated_records = []
base_path = [] if self.parent is None else list(self.parent.path)
new_path = [*base_path, str(self.pk)]
path_has_changed = new_path != self.path
if path_has_changed:
self.path = new_path
updated_records += [self]
if path_has_changed or force_recalculate:
for child in self.orgunit_set.all():
updated_records += child.calculate_paths(force_recalculate)
return updated_records
def __str__(self):
return "%s %s %d" % (self.org_unit_type, self.name,
self.id if self.id else -1)
def as_dict_for_mobile_lite(self):
return {
"n": self.name,
"id": self.id,
"p": self.parent_id,
"out": self.org_unit_type_id,
"c_a": self.created_at.timestamp() if self.created_at else None,
"lat": self.location.y if self.location else None,
"lon": self.location.x if self.location else None,
"alt": self.location.z if self.location else None,
}
def as_dict_for_mobile(self):
return {
"name":
self.name,
"id":
self.id,
"parent_id":
self.parent_id,
"org_unit_type_id":
self.org_unit_type_id,
"org_unit_type_name":
self.org_unit_type.name if self.org_unit_type else None,
"validation_status":
self.validation_status if self.org_unit_type else None,
"created_at":
self.created_at.timestamp() if self.created_at else None,
"updated_at":
self.updated_at.timestamp() if self.updated_at else None,
"latitude":
self.location.y if self.location else None,
"longitude":
self.location.x if self.location else None,
"altitude":
self.location.z if self.location else None,
}
def as_dict(self, with_groups=True):
res = {
"name":
self.name,
"short_name":
self.name,
"id":
self.id,
"source":
self.version.data_source.name if self.version else None,
"source_ref":
self.source_ref,
"parent_id":
self.parent_id,
"org_unit_type_id":
self.org_unit_type_id,
"org_unit_type_name":
self.org_unit_type.name if self.org_unit_type else None,
"created_at":
self.created_at.timestamp() if self.created_at else None,
"updated_at":
self.updated_at.timestamp() if self.updated_at else None,
"aliases":
self.aliases,
"validation_status":
self.validation_status,
"latitude":
self.location.y if self.location else None,
"longitude":
self.location.x if self.location else None,
"altitude":
self.location.z if self.location else None,
"has_geo_json":
True if self.simplified_geom else False,
"version":
self.version.number if self.version else None,
}
if hasattr(self, "search_index"):
res["search_index"] = self.search_index
return res
def as_dict_with_parents(self, light=False, light_parents=True):
res = {
"name":
self.name,
"short_name":
self.name,
"id":
self.id,
"sub_source":
self.sub_source,
"sub_source_id":
self.sub_source,
"source_ref":
self.source_ref,
"source_url":
self.version.data_source.credentials.url
if self.version and self.version.data_source
and self.version.data_source.credentials else None,
"parent_id":
self.parent_id,
"validation_status":
self.validation_status,
"parent_name":
self.parent.name if self.parent else None,
"parent":
self.parent.as_dict_with_parents(light=light_parents,
light_parents=light_parents)
if self.parent else None,
"org_unit_type_id":
self.org_unit_type_id,
"created_at":
self.created_at.timestamp() if self.created_at else None,
"updated_at":
self.updated_at.timestamp() if self.updated_at else None,
"aliases":
self.aliases,
"latitude":
self.location.y if self.location else None,
"longitude":
self.location.x if self.location else None,
"altitude":
self.location.z if self.location else None,
"has_geo_json":
True if self.simplified_geom else False,
}
if not light: # avoiding joins here
res["groups"] = [
group.as_dict(with_counts=False)
for group in self.groups.all()
]
res["org_unit_type_name"] = self.org_unit_type.name if self.org_unit_type else None
res["org_unit_type"] = self.org_unit_type.as_dict(
) if self.org_unit_type else None
res["source"] = self.version.data_source.name if self.version else None
res["source_id"] = self.version.data_source.id if self.version else None
res["version"] = self.version.number if self.version else None
if hasattr(self, "search_index"):
res["search_index"] = self.search_index
return res
def as_small_dict(self):
res = {
"name":
self.name,
"id":
self.id,
"parent_id":
self.parent_id,
"validation_status":
self.validation_status,
"parent_name":
self.parent.name if self.parent else None,
"source":
self.version.data_source.name if self.version else None,
"source_ref":
self.source_ref,
"parent":
self.parent.as_small_dict() if self.parent else None,
"org_unit_type_name":
self.org_unit_type.name if self.org_unit_type else None,
}
if hasattr(self, "search_index"):
res["search_index"] = self.search_index
return res
def as_dict_for_csv(self):
return {
"name": self.name,
"id": self.id,
"source_ref": self.source_ref,
"parent_id": self.parent_id,
"org_unit_type": self.org_unit_type.name,
}
def as_location(self):
res = {
"id":
self.id,
"name":
self.name,
"short_name":
self.name,
"latitude":
self.location.y if self.location else None,
"longitude":
self.location.x if self.location else None,
"altitude":
self.location.z if self.location else None,
"has_geo_json":
True if self.simplified_geom else False,
"org_unit_type":
self.org_unit_type.name if self.org_unit_type else None,
"org_unit_type_depth":
self.org_unit_type.depth if self.org_unit_type else None,
"source_id":
self.version.data_source.id if self.version else None,
"source_name":
self.version.data_source.name if self.version else None,
}
if hasattr(self, "search_index"):
res["search_index"] = self.search_index
return res
def source_path(self):
"""DHIS2-friendly path built using source refs"""
path_components = []
cur = self
while cur:
if cur.source_ref:
path_components.insert(0, cur.source_ref)
cur = cur.parent
if len(path_components) > 0:
return "/" + ("/".join(path_components))
return None