class Checksum(db.EmbeddedDocument):
type = db.StringField(choices=CHECKSUM_TYPES, required=True)
value = db.StringField(required=True)
def to_mongo(self, *args, **kwargs):
if bool(self.value):
return super(Checksum, self).to_mongo()
class HarvestSource(db.Document):
name = db.StringField(max_length=255)
slug = db.SlugField(max_length=255,
required=True,
unique=True,
populate_from='name',
update=True)
description = db.StringField()
url = db.StringField()
backend = db.StringField()
config = db.DictField()
periodic_task = db.ReferenceField('PeriodicTask',
reverse_delete_rule=db.NULLIFY)
created_at = db.DateTimeField(default=datetime.now, required=True)
frequency = db.StringField(choices=HARVEST_FREQUENCIES.keys(),
default=DEFAULT_HARVEST_FREQUENCY,
required=True)
active = db.BooleanField(default=True)
owner = db.ReferenceField('User', reverse_delete_rule=db.NULLIFY)
organization = db.ReferenceField('Organization',
reverse_delete_rule=db.NULLIFY)
@classmethod
def get(cls, ident):
return cls.objects(slug=ident).first() or cls.objects.get(id=ident)
def get_last_job(self):
return HarvestJob.objects(source=self).order_by('-created')[0]
class Role(db.Document, RoleMixin):
ADMIN = 'admin'
name = db.StringField(max_length=80, unique=True)
description = db.StringField(max_length=255)
def __str__(self):
return self.name
class OAuth2Token(db.Document):
client = db.ReferenceField('OAuth2Client', required=True)
user = db.ReferenceField('User')
# currently only bearer is supported
token_type = db.StringField(choices=TOKEN_TYPES.keys(), default='Bearer')
access_token = db.StringField(unique=True)
refresh_token = db.StringField(unique=True, sparse=True)
created_at = db.DateTimeField(default=datetime.utcnow, required=True)
expires_in = db.IntField(required=True, default=TOKEN_EXPIRATION)
scopes = db.ListField(db.StringField())
meta = {'collection': 'oauth2_token'}
def __unicode__(self):
return '<OAuth2Token({0.client.name})>'.format(self)
def get_scope(self):
return ' '.join(self.scopes)
def get_expires_in(self):
return self.expires_in
def get_expires_at(self):
return (self.created_at - EPOCH).total_seconds() + self.expires_in
def is_refresh_token_expired(self):
expired_at = datetime.fromtimestamp(self.get_expires_at())
expired_at += timedelta(days=REFRESH_EXPIRATION)
return expired_at < datetime.utcnow()
class Topic(db.Document):
name = db.StringField(max_length=255, required=True)
slug = db.SlugField(max_length=255,
required=True,
populate_from='name',
update=True)
description = db.StringField()
tags = db.ListField(db.StringField())
color = db.IntField()
tags = db.ListField(db.StringField())
datasets = db.ListField(
db.ReferenceField('Dataset', reverse_delete_rule=db.PULL))
reuses = db.ListField(
db.ReferenceField('Reuse', reverse_delete_rule=db.PULL))
owner = db.ReferenceField('User')
featured = db.BooleanField()
private = db.BooleanField()
def __unicode__(self):
return self.name
@property
def display_url(self):
return url_for('topics.display', topic=self)
class OAuth2Grant(db.Document):
user = db.ReferenceField('User', required=True)
client = db.ReferenceField('OAuth2Client', required=True)
code = db.StringField(required=True)
redirect_uri = db.StringField()
expires = db.DateTimeField()
scopes = db.ListField(db.StringField())
meta = {
'collection': 'oauth2_grant'
}
def __str__(self):
return '<OAuth2Grant({0.client.name}, {0.user.fullname})>'.format(self)
def is_expired(self):
return self.expires < datetime.utcnow()
def get_redirect_uri(self):
return self.redirect_uri
def get_scope(self):
return ' '.join(self.scopes)
class GeoLevel(db.Document):
id = db.StringField(primary_key=True)
name = db.StringField(required=True)
admin_level = db.IntField(min_value=ADMIN_LEVEL_MIN,
max_value=ADMIN_LEVEL_MAX,
default=100)
parents = db.ListField(db.ReferenceField('self'))
class OAuth2Code(db.Document):
user = db.ReferenceField('User', required=True)
client = db.ReferenceField('OAuth2Client', required=True)
code = db.StringField(required=True)
redirect_uri = db.StringField()
expires = db.DateTimeField()
scope = db.StringField(default='')
code_challenge = db.StringField()
code_challenge_method = db.StringField()
meta = {'collection': 'oauth2_code'}
def __str__(self):
return '<OAuth2Code({0.client.name}, {0.user.fullname})>'.format(self)
def is_expired(self):
return self.expires < datetime.utcnow()
def get_redirect_uri(self):
return self.redirect_uri
def get_scope(self):
return self.scope
class Metrics(db.Document):
object_id = db.DynamicField(required=True, null=False, unique_with='date')
date = db.StringField(required=True)
level = db.StringField(
required=True, default='daily',
choices=('hourly', 'daily', 'monthly', 'yearly'))
values = db.DictField()
meta = {
'indexes': [
'object_id',
'level',
'date',
{
'fields': ('object_id', 'level', '-date'),
'unique': True,
}
],
'queryset_class': MetricsQuerySet,
}
def __unicode__(self):
return 'Metrics for object {0} on {1} ({2})'.format(
self.object_id, self.date, self.level
)
class Site(WithMetrics, db.Document):
id = db.StringField(primary_key=True)
title = db.StringField(required=True)
keywords = db.ListField(db.StringField())
feed_size = db.IntField(required=True, default=DEFAULT_FEED_SIZE)
configs = db.DictField()
themes = db.DictField()
settings = db.EmbeddedDocumentField(SiteSettings, default=SiteSettings)
class Fake(db.Document):
title = db.StringField()
description = db.StringField()
tags = db.ListField(db.StringField())
other = db.ListField(db.StringField())
def __unicode__(self):
return 'fake'
class TerritoryReference(db.EmbeddedDocument):
id = db.ObjectIdField(required=True)
name = db.StringField(required=True)
level = db.StringField(required=True)
code = db.StringField(required=True)
def __unicode__(self):
return self.name
class HarvestSourceValidation(db.EmbeddedDocument):
'''Store harvest source validation details'''
state = db.StringField(choices=VALIDATION_STATES.keys(),
default=VALIDATION_PENDING,
required=True)
by = db.ReferenceField('User')
on = db.DateTimeField()
comment = db.StringField()
class Team(db.EmbeddedDocument):
name = db.StringField(required=True)
slug = db.SlugField(
max_length=255, required=True, populate_from='name', update=True,
unique=False)
description = db.StringField()
members = db.ListField(db.ReferenceField('User'))
class License(db.Document):
# We need to declare id explicitly since we do not use the default
# value set by Mongo.
id = db.StringField(primary_key=True)
created_at = db.DateTimeField(default=datetime.now, required=True)
title = db.StringField(required=True)
slug = db.SlugField(required=True, populate_from='title')
url = db.URLField()
maintainer = db.StringField()
flags = db.ListField(db.StringField())
active = db.BooleanField()
def __unicode__(self):
return self.title
@classmethod
def guess(cls, *strings, **kwargs):
'''
Try to guess a license from a list of strings.
Accept a `default` keyword argument which will be
the default fallback license.
'''
license = None
for string in strings:
license = cls.guess_one(string)
if license:
break
return license or kwargs.get('default')
@classmethod
def guess_one(cls, text):
'''
Try to guess license from a string.
Try to exact match on identifier then slugified title
and fallback on edit distance ranking (after slugification)
'''
if not text:
return
qs = cls.objects
text = text.strip().lower() # Stored identifiers are lower case
slug = cls.slug.slugify(text) # Use slug as it normalize string
license = qs(db.Q(id=text) | db.Q(slug=slug) | db.Q(url=text)).first()
if license is None:
# Try to single match with a low Damerau-Levenshtein distance
computed = ((l, rdlevenshtein(l.slug, slug)) for l in cls.objects)
candidates = [l for l, d in computed if d <= MAX_DISTANCE]
# If there is more that one match, we cannot determinate
# which one is closer to safely choose between candidates
if len(candidates) == 1:
license = candidates[0]
return license
@classmethod
def default(cls):
return cls.objects(id=DEFAULT_LICENSE['id']).first()
class HarvestSource(db.Owned, db.Document):
name = db.StringField(max_length=255)
slug = db.SlugField(max_length=255,
required=True,
unique=True,
populate_from='name',
update=True)
description = db.StringField()
url = db.StringField(required=True)
backend = db.StringField()
config = db.DictField()
periodic_task = db.ReferenceField('PeriodicTask',
reverse_delete_rule=db.NULLIFY)
created_at = db.DateTimeField(default=datetime.now, required=True)
frequency = db.StringField(choices=HARVEST_FREQUENCIES.keys(),
default=DEFAULT_HARVEST_FREQUENCY,
required=True)
active = db.BooleanField(default=True)
validation = db.EmbeddedDocumentField(HarvestSourceValidation,
default=HarvestSourceValidation)
deleted = db.DateTimeField()
@property
def domain(self):
parsed = urlparse(self.url)
return parsed.netloc.split(':')[0]
@classmethod
def get(cls, ident):
return cls.objects(slug=ident).first() or cls.objects.get(pk=ident)
def get_last_job(self):
return HarvestJob.objects(source=self).order_by('-created').first()
@cached_property
def last_job(self):
return self.get_last_job()
@property
def schedule(self):
if not self.periodic_task:
return
return self.periodic_task.schedule_display
meta = {
'indexes': [
'-created_at',
'slug',
('deleted', '-created_at'),
] + db.Owned.meta['indexes'],
'ordering': ['-created_at'],
'queryset_class':
HarvestSourceQuerySet,
}
def __unicode__(self):
return self.name or ''
class OrgUnit(object):
'''
Simple mixin holding common fields for all organization units.
'''
name = db.StringField(max_length=255, required=True)
slug = db.SlugField(max_length=255, required=True, populate_from='name', update=True)
description = db.StringField(required=True)
url = db.URLField(max_length=255)
image_url = db.URLField(max_length=255)
extras = db.DictField()
class Fake(db.Document):
title = db.StringField()
description = db.StringField()
tags = db.ListField(db.StringField())
other = db.ListField(db.StringField())
nested = db.ListField(db.EmbeddedDocumentField(NestedFake))
metrics = db.DictField()
def __unicode__(self):
return 'fake'
class Fake(db.Document):
title = db.StringField()
description = db.StringField()
tags = db.ListField(db.StringField())
other = db.ListField(db.StringField())
meta = {'allow_inheritance': True}
def __unicode__(self):
return self.title
class HarvestItem(db.EmbeddedDocument):
remote_id = db.StringField()
dataset = db.ReferenceField(Dataset)
status = db.StringField(choices=HARVEST_ITEM_STATUS.keys(),
default=DEFAULT_HARVEST_ITEM_STATUS, required=True)
created = db.DateTimeField(default=datetime.now, required=True)
started = db.DateTimeField()
ended = db.DateTimeField()
errors = db.ListField(db.EmbeddedDocumentField(HarvestError))
args = db.ListField(db.StringField())
kwargs = db.DictField()
class OAuth2Grant(db.Document):
user = db.ReferenceField('User', required=True)
client = db.ReferenceField('OAuth2Client', required=True)
code = db.StringField(required=True)
redirect_uri = db.StringField()
expires = db.DateTimeField()
scopes = db.ListField(db.StringField())
meta = {'collection': 'oauth2_grant'}
class GeoZone(db.Document):
id = db.StringField(primary_key=True)
name = db.StringField(required=True)
level = db.StringField(required=True)
code = db.StringField(unique_with='level')
geom = db.MultiPolygonField(required=True)
parents = db.ListField()
keys = db.DictField()
population = db.IntField()
area = db.FloatField()
meta = {
'indexes': [
'name',
'parents',
('level', 'code'),
]
}
def __unicode__(self):
return self.id
__str__ = __unicode__
def __html__(self):
return gettext(self.name) + ' <i>(' + self.code + ')</i>'
@property
def keys_values(self):
"""Key values might be a list or not, always return a list."""
keys_values = []
for value in self.keys.values():
if isinstance(value, list):
keys_values += value
elif not str(value).startswith('-'): # Avoid -99.
keys_values.append(value)
return keys_values
def toGeoJSON(self):
return {
'id': self.id,
'type': 'Feature',
'geometry': self.geom,
'properties': {
'name': self.name,
'level': self.level,
'code': self.code,
'parents': self.parents,
'keys': self.keys,
'population': self.population,
'area': self.area,
}
}
class License(db.Document):
id = db.StringField(primary_key=True)
created_at = db.DateTimeField(default=datetime.datetime.now, required=True)
title = db.StringField(required=True)
slug = db.SlugField(required=True, populate_from='title')
url = db.URLField()
maintainer = db.StringField()
flags = db.ListField(db.StringField())
active = db.BooleanField()
def __unicode__(self):
return self.title
class OAuth2Client(db.Datetimed, db.Document):
secret = db.StringField(default=lambda: gen_salt(50), required=True)
type = db.StringField(choices=CLIENT_TYPES.keys(), default='public', required=True)
profile = db.StringField(choices=CLIENT_PROFILES.keys(), default='web', required=True)
grant_type = db.StringField(choices=GRANT_TYPES.keys(), default='code', required=True)
name = db.StringField(required=True)
description = db.StringField()
owner = db.ReferenceField('User')
organization = db.ReferenceField('Organization')
image = db.ImageField(fs=images, basename=default_image_basename, thumbnails=[150, 25])
redirect_uris = db.ListField(db.StringField())
default_scopes = db.ListField(db.StringField(), default=SCOPES.keys())
meta = {
'collection': 'oauth2_client'
}
@property
def client_id(self):
return str(self.id)
@property
def client_secret(self):
return self.secret
@property
def default_redirect_uri(self):
return self.redirect_uris[0]
class OAuth2Token(db.Document):
client = db.ReferenceField('OAuth2Client', required=True)
user = db.ReferenceField('User')
# currently only bearer is supported
token_type = db.StringField(choices=TOKEN_TYPES.keys(), default='Bearer')
access_token = db.StringField(unique=True)
refresh_token = db.StringField(unique=True)
expires = db.DateTimeField(
default=lambda: datetime.utcnow() + timedelta(hours=TOKEN_EXPIRATION))
scopes = db.ListField(db.StringField())
meta = {'collection': 'oauth2_token'}
class License(db.Document):
# We need to declare id explicitly since we do not use the default
# value set by Mongo.
id = db.StringField(primary_key=True)
created_at = db.DateTimeField(default=datetime.now, required=True)
title = db.StringField(required=True)
slug = db.SlugField(required=True, populate_from='title')
url = db.URLField()
maintainer = db.StringField()
flags = db.ListField(db.StringField())
active = db.BooleanField()
def __unicode__(self):
return self.title
class SpatialCoverage(db.EmbeddedDocument):
"""Represent a spatial coverage as a list of territories and/or a geometry.
"""
geom = db.MultiPolygonField()
zones = db.ListField(db.ReferenceField(GeoZone))
granularity = db.StringField(default='other')
@property
def granularity_label(self):
return _(dict(spatial_granularities)[self.granularity or 'other'])
@property
def top_label(self):
if not self.zones:
return None
top = None
for zone in self.zones:
if not top:
top = zone
continue
if zone.id in top.parents:
top = zone
return _(top.name)
@property
def handled_zones(self):
"""Return only zones with a dedicated page."""
return [zone for zone in self.zones if zone.handled_level]
class Fake(db.Document):
title = db.StringField()
description = db.StringField()
tags = db.ListField(db.StringField())
other = db.ListField(db.StringField())
meta = {'allow_inheritance': True}
def __unicode__(self):
return self.title
def __str__(self):
return self.title
def __html__(self):
return '<span>{0}</span>'.format(self.title)
class PeriodicTask(BasePeriodicTask):
last_run_id = db.StringField()
class Interval(BasePeriodicTask.Interval):
def __str__(self):
if self.every == 1:
return _('every {0.period_singular}').format(self)
return _('every {0.every} {0.period}').format(self)
class Crontab(BasePeriodicTask.Crontab):
def __str__(self):
return CRON.format(**self._data)
@classmethod
def parse(cls, cron):
m, h, d, M, W = cron.split()
return cls(
minute=m,
hour=h,
day_of_month=d,
month_of_year=M,
day_of_week=W,
)
@property
def schedule_display(self):
if self.interval:
return str(self.interval)
elif self.crontab:
return str(self.crontab)
else:
raise Exception("must define internal or crontab schedule")
interval = db.EmbeddedDocumentField(Interval)
crontab = db.EmbeddedDocumentField(Crontab)
class Discussion(db.Document):
user = db.ReferenceField('User')
subject = db.GenericReferenceField()
title = db.StringField(required=True)
discussion = db.ListField(db.EmbeddedDocumentField(Message))
created = db.DateTimeField(default=datetime.now, required=True)
closed = db.DateTimeField()
closed_by = db.ReferenceField('User')
extras = db.ExtrasField()
meta = {
'indexes': [
'user',
'subject',
'-created'
],
'ordering': ['-created'],
}
def person_involved(self, person):
"""Return True if the given person has been involved in the
discussion, False otherwise.
"""
return any(message.posted_by == person for message in self.discussion)
@property
def external_url(self):
return self.subject.url_for(
_anchor='discussion-{id}'.format(id=self.id),
_external=True)
