class Resource(WithMetrics, db.EmbeddedDocument): id = db.AutoUUIDField() title = db.StringField(verbose_name="Title", required=True) description = db.StringField() type = db.StringField(choices=RESOURCE_TYPES.keys(), default='file', required=True) url = db.StringField() urlhash = db.StringField() checksum = db.EmbeddedDocumentField(Checksum) format = db.StringField() mime = db.StringField() size = db.IntField() owner = db.ReferenceField('User') created_at = db.DateTimeField(default=datetime.datetime.now, required=True) modified = db.DateTimeField(default=datetime.datetime.now, required=True) published = db.DateTimeField(default=datetime.datetime.now, required=True) deleted = db.DateTimeField() on_added = Signal() on_deleted = Signal() def clean(self): super(Resource, self).clean() if not self.urlhash or 'url' in self._get_changed_fields(): self.urlhash = hash_url(self.url)
class GeoLevel(db.Document): id = db.StringField(primary_key=True) name = db.StringField(required=True) admin_level = db.IntField(min_value=ADMIN_LEVEL_MIN, max_value=ADMIN_LEVEL_MAX, default=100) parents = db.ListField(db.ReferenceField('self'))
class Topic(db.Document): name = db.StringField(max_length=255, required=True) slug = db.SlugField(max_length=255, required=True, populate_from='name', update=True) description = db.StringField() tags = db.ListField(db.StringField()) color = db.IntField() tags = db.ListField(db.StringField()) datasets = db.ListField( db.ReferenceField('Dataset', reverse_delete_rule=db.PULL)) reuses = db.ListField( db.ReferenceField('Reuse', reverse_delete_rule=db.PULL)) owner = db.ReferenceField('User') featured = db.BooleanField() private = db.BooleanField() def __unicode__(self): return self.name @property def display_url(self): return url_for('topics.display', topic=self)
class OAuth2Token(db.Document): client = db.ReferenceField('OAuth2Client', required=True) user = db.ReferenceField('User') # currently only bearer is supported token_type = db.StringField(choices=TOKEN_TYPES.keys(), default='Bearer') access_token = db.StringField(unique=True) refresh_token = db.StringField(unique=True, sparse=True) created_at = db.DateTimeField(default=datetime.utcnow, required=True) expires_in = db.IntField(required=True, default=TOKEN_EXPIRATION) scopes = db.ListField(db.StringField()) meta = {'collection': 'oauth2_token'} def __unicode__(self): return '<OAuth2Token({0.client.name})>'.format(self) def get_scope(self): return ' '.join(self.scopes) def get_expires_in(self): return self.expires_in def get_expires_at(self): return (self.created_at - EPOCH).total_seconds() + self.expires_in def is_refresh_token_expired(self): expired_at = datetime.fromtimestamp(self.get_expires_at()) expired_at += timedelta(days=REFRESH_EXPIRATION) return expired_at < datetime.utcnow()
class Site(WithMetrics, db.Document): id = db.StringField(primary_key=True) title = db.StringField(required=True) keywords = db.ListField(db.StringField()) feed_size = db.IntField(required=True, default=DEFAULT_FEED_SIZE) configs = db.DictField() themes = db.DictField() settings = db.EmbeddedDocumentField(SiteSettings, default=SiteSettings)
class GeoZone(db.Document): id = db.StringField(primary_key=True) name = db.StringField(required=True) level = db.StringField(required=True) code = db.StringField(unique_with='level') geom = db.MultiPolygonField(required=True) parents = db.ListField() keys = db.DictField() population = db.IntField() area = db.FloatField() meta = { 'indexes': [ 'name', 'parents', ('level', 'code'), ] } def __unicode__(self): return self.id __str__ = __unicode__ def __html__(self): return gettext(self.name) + ' <i>(' + self.code + ')</i>' @property def keys_values(self): """Key values might be a list or not, always return a list.""" keys_values = [] for value in self.keys.values(): if isinstance(value, list): keys_values += value elif not str(value).startswith('-'): # Avoid -99. keys_values.append(value) return keys_values def toGeoJSON(self): return { 'id': self.id, 'type': 'Feature', 'geometry': self.geom, 'properties': { 'name': self.name, 'level': self.level, 'code': self.code, 'parents': self.parents, 'keys': self.keys, 'population': self.population, 'area': self.area, } }
class Tag(db.Document): name = db.StringField(required=True, unique=True) counts = db.DictField() total = db.IntField(default=0) meta = { 'indexes': ['name', '-total'], 'ordering': [ '-total', ], } def clean(self): self.total = sum(self.counts.values())
class Tag(db.Document): ''' This collection is auto-populated every hour map-reducing tag properties from Datasets dans Reuses. ''' name = db.StringField(required=True, unique=True) counts = db.DictField() total = db.IntField(default=0) meta = { 'indexes': ['name', '-total'], 'ordering': ['-total', ], } def clean(self): self.total = sum(self.counts.values())
class ResourceMixin(object): id = db.AutoUUIDField(primary_key=True) title = db.StringField(verbose_name="Title", required=True) description = db.StringField() filetype = db.StringField(choices=RESOURCE_TYPES.keys(), default='file', required=True) url = db.StringField() urlhash = db.StringField() checksum = db.EmbeddedDocumentField(Checksum) format = db.StringField() mime = db.StringField() filesize = db.IntField() # `size` is a reserved keyword for mongoengine. created_at = db.DateTimeField(default=datetime.now, required=True) modified = db.DateTimeField(default=datetime.now, required=True) published = db.DateTimeField(default=datetime.now, required=True) deleted = db.DateTimeField() def clean(self): super(ResourceMixin, self).clean() if not self.urlhash or 'url' in self._get_changed_fields(): self.urlhash = hash_url(self.url) @property def closed_format(self): """Return True if the specified format is in CLOSED_FORMATS.""" return self.format.lower() in CLOSED_FORMATS def check_availability(self, group): """Check if a resource is reachable against a Croquemort server. Return a boolean. """ if self.filetype == 'remote': # We perform a quick check for performances matters. error, response = check_url_from_cache(self.url, group) if error or int(response.get('status', 500)) >= 500: return False else: return True else: return True # We consider that API cases (types) are OK. @property def is_available(self): return self.check_availability(group=None)
class GeoZone(db.Document): id = db.StringField(primary_key=True) name = db.StringField(required=True) level = db.StringField(required=True) code = db.StringField(unique_with='level') geom = db.MultiPolygonField(required=True) parents = db.ListField() keys = db.DictField() population = db.IntField() area = db.FloatField() meta = { 'indexes': [ 'name', 'parents', ('level', 'code'), ] } def __unicode__(self): return self.id __str__ = __unicode__ def __html__(self): return gettext(self.name) + ' <i>(' + self.code + ')</i>' def toGeoJSON(self): return { 'id': self.id, 'type': 'Feature', 'geometry': self.geom, 'properties': { 'name': self.name, 'level': self.level, 'code': self.code, 'parents': self.parents, 'keys': self.keys, 'population': self.population, 'area': self.area, } }
class User(db.Document, WithMetrics, UserMixin): slug = db.SlugField(max_length=255, required=True, populate_from='fullname') email = db.StringField(max_length=255, required=True) password = db.StringField() active = db.BooleanField() roles = db.ListField(db.ReferenceField(Role), default=[]) first_name = db.StringField(max_length=255, required=True) last_name = db.StringField(max_length=255, required=True) avatar_url = db.URLField() avatar = db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES) website = db.URLField() about = db.StringField() prefered_language = db.StringField() apikey = db.StringField() created_at = db.DateTimeField(default=datetime.now, required=True) confirmed_at = db.DateTimeField() last_login_at = db.DateTimeField() current_login_at = db.DateTimeField() last_login_ip = db.StringField() current_login_ip = db.StringField() login_count = db.IntField() deleted = db.DateTimeField() ext = db.MapField(db.GenericEmbeddedDocumentField()) extras = db.ExtrasField() before_save = Signal() after_save = Signal() on_create = Signal() on_update = Signal() before_delete = Signal() after_delete = Signal() on_delete = Signal() meta = { 'allow_inheritance': True, 'indexes': ['-created_at', 'slug', 'apikey'], 'ordering': ['-created_at'] } def __str__(self): return self.fullname __unicode__ = __str__ @property def fullname(self): return ' '.join((self.first_name or '', self.last_name or '')).strip() @cached_property def organizations(self): from udata.core.organization.models import Organization return Organization.objects(members__user=self) @property def sysadmin(self): return self.has_role('admin') @property def display_url(self): return url_for('users.show', user=self) @property def visible(self): return (self.metrics.get('datasets', 0) + self.metrics.get('reuses', 0)) > 0 def generate_api_key(self): s = JSONWebSignatureSerializer(current_app.config['SECRET_KEY']) self.apikey = s.dumps({ 'user': str(self.id), 'time': time(), }) def clear_api_key(self): self.apikey = None @classmethod def get(cls, id_or_slug): obj = cls.objects(slug=id_or_slug).first() return obj or cls.objects.get_or_404(id=id_or_slug) @classmethod def pre_save(cls, sender, document, **kwargs): cls.before_save.send(document) @classmethod def post_save(cls, sender, document, **kwargs): cls.after_save.send(document) if kwargs.get('created'): cls.on_create.send(document) else: cls.on_update.send(document)
class GeoZone(db.Document): id = db.StringField(primary_key=True) name = db.StringField(required=True) level = db.StringField(required=True) code = db.StringField(unique_with='level') geom = db.MultiPolygonField(required=True) parents = db.ListField() keys = db.DictField() population = db.IntField() area = db.FloatField() wikipedia = db.StringField() dbpedia = db.StringField() logo = db.ImageField(fs=logos) meta = { 'indexes': [ 'name', 'parents', ('level', 'code'), ] } def __unicode__(self): return self.id __str__ = __unicode__ def __html__(self): return '{name} <i>({code})</i>'.format( name=gettext(self.name), code=self.code) def logo_url(self, external=False): filename = self.logo.filename if filename and self.logo.fs.exists(filename): return self.logo.fs.url(filename, external=external) else: return '' @property def keys_values(self): """Key values might be a list or not, always return a list.""" keys_values = [] for value in self.keys.values(): if isinstance(value, list): keys_values += value elif not str(value).startswith('-'): # Avoid -99. keys_values.append(value) return keys_values @property def url(self): return url_for('territories.territory', territory=self) @property def external_url(self): return url_for('territories.territory', territory=self, _external=True) @cached_property def wikipedia_url(self): return (self.dbpedia.replace('dbpedia', 'wikipedia') .replace('resource', 'wiki')) @cached_property def postal_string(self): """Return a list of postal codes separated by commas.""" return ', '.join(self.keys.get('postal', [])) @cached_property def town_repr(self): """Representation of a town with optional county.""" if self.county: return '{name} <small>({county_name})</small>'.format( name=self.name, county_name=self.county.name) return self.name @cached_property def county(self): for parent in self.parents: if parent.startswith('fr/county'): return GeoZone.objects.get(id=parent) def toGeoJSON(self): return { 'id': self.id, 'type': 'Feature', 'geometry': self.geom, 'properties': { 'name': _(self.name), 'level': self.level, 'code': self.code, 'parents': self.parents, 'keys': self.keys, 'population': self.population, 'area': self.area, } }
class GeoZone(db.Document): id = db.StringField(primary_key=True) name = db.StringField(required=True) level = db.StringField(required=True) code = db.StringField(unique_with='level') geom = db.MultiPolygonField(required=True) parents = db.ListField() keys = db.DictField() population = db.IntField() area = db.FloatField() wikipedia = db.StringField() dbpedia = db.StringField() logo = db.ImageField(fs=logos) meta = { 'indexes': [ 'name', 'parents', ('level', 'code'), ] } def __unicode__(self): return self.id __str__ = __unicode__ def __html__(self): """In use within the admin.""" return '{name} <i>({code})</i>'.format(name=gettext(self.name), code=self.code) @cached_property def html_title(self): """In use within templates.""" if self.level_name == 'town': return ('{name} ' '<small>(<a href="{parent_url}">{parent_name}</a>)</small>' '').format(name=self.name, parent_url=self.parent.url, parent_name=self.parent.name) elif self.level_name == 'county': return '{name} <small>({code})</small>'.format(name=self.name, code=self.code) else: return self.name def logo_url(self, external=False): filename = self.logo.filename if filename and self.logo.fs.exists(filename): return self.logo.fs.url(filename, external=external) else: return '' @property def keys_values(self): """Key values might be a list or not, always return a list.""" keys_values = [] for value in self.keys.values(): if isinstance(value, list): keys_values += value elif not str(value).startswith('-'): # Avoid -99. keys_values.append(value) return keys_values @cached_property def level_name(self): """Truncated level name for the sake of readability.""" if self.level.startswith('fr/'): return self.level[3:] # Keep the whole level name as a fallback (e.g. `country/fr`) return self.level @cached_property def level_i18n_name(self): """In use within templates for dynamic translations.""" for level, name in spatial_granularities: if self.level == level: return name return self.level_name # Fallback that should never happen. @cached_property def child_level(self): """Return the child level given handled levels.""" HANDLED_LEVELS = current_app.config.get('HANDLED_LEVELS') try: return HANDLED_LEVELS[HANDLED_LEVELS.index(self.level) - 1] except (IndexError, ValueError): return None @cached_property def parent_level(self): """Return the parent level given handled levels.""" HANDLED_LEVELS = current_app.config.get('HANDLED_LEVELS') try: return HANDLED_LEVELS[HANDLED_LEVELS.index(self.level) + 1] except (IndexError, ValueError): return None @property def url(self): return url_for('territories.territory', territory=self) @property def external_url(self): return url_for('territories.territory', territory=self, _external=True) @cached_property def wikipedia_url(self): """Computed wikipedia URL from the DBpedia one.""" return (self.dbpedia.replace('dbpedia', 'wikipedia').replace('resource', 'wiki')) @cached_property def postal_string(self): """Return a list of postal codes separated by commas.""" return ', '.join(self.keys.get('postal', [])) @property def parent(self): if self.parent_level: for parent in self.parents: if parent.startswith(self.parent_level): return GeoZone.objects.get(id=parent, level=self.parent_level) @property def children(self): if self.child_level: return (GeoZone.objects(level=self.child_level, parents__in=[self.id]).order_by( '-population', '-area')) @property def handled_level(self): return self.level in current_app.config.get('HANDLED_LEVELS') def toGeoJSON(self): return { 'id': self.id, 'type': 'Feature', 'geometry': self.geom, 'properties': { 'name': gettext(self.name), 'level': self.level, 'code': self.code, 'parents': self.parents, 'keys': self.keys, 'population': self.population, 'area': self.area, 'logo': self.logo_url(external=True) } }
class User(WithMetrics, UserMixin, db.Document): slug = db.SlugField(max_length=255, required=True, populate_from='fullname') email = db.StringField(max_length=255, required=True, unique=True) password = db.StringField() active = db.BooleanField() roles = db.ListField(db.ReferenceField(Role), default=[]) first_name = db.StringField(max_length=255, required=True) last_name = db.StringField(max_length=255, required=True) avatar_url = db.URLField() avatar = db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES) website = db.URLField() about = db.StringField() prefered_language = db.StringField() apikey = db.StringField() created_at = db.DateTimeField(default=datetime.now, required=True) # The field below is required for Flask-security # when SECURITY_CONFIRMABLE is True confirmed_at = db.DateTimeField() # The 5 fields below are required for Flask-security # when SECURITY_TRACKABLE is True last_login_at = db.DateTimeField() current_login_at = db.DateTimeField() last_login_ip = db.StringField() current_login_ip = db.StringField() login_count = db.IntField() deleted = db.DateTimeField() ext = db.MapField(db.GenericEmbeddedDocumentField()) extras = db.ExtrasField() before_save = Signal() after_save = Signal() on_create = Signal() on_update = Signal() before_delete = Signal() after_delete = Signal() on_delete = Signal() meta = { 'indexes': ['-created_at', 'slug', 'apikey'], 'ordering': ['-created_at'] } def __str__(self): return self.fullname __unicode__ = __str__ @property def fullname(self): return ' '.join((self.first_name or '', self.last_name or '')).strip() @cached_property def organizations(self): from udata.core.organization.models import Organization return Organization.objects(members__user=self, deleted__exists=False) @property def sysadmin(self): return self.has_role('admin') def url_for(self, *args, **kwargs): return url_for('users.show', user=self, *args, **kwargs) display_url = property(url_for) @property def external_url(self): return self.url_for(_external=True) @property def visible(self): count = self.metrics.get('datasets', 0) + self.metrics.get('reuses', 0) return count > 0 and self.active @cached_property def resources_availability(self): """Return the percentage of availability for resources.""" # Flatten the list. availabilities = list( chain(*[org.check_availability() for org in self.organizations])) if availabilities: # Trick will work because it's a sum() of booleans. return round(100. * sum(availabilities) / len(availabilities), 2) else: return 0 @cached_property def datasets_org_count(self): """Return the number of datasets of user's organizations.""" from udata.models import Dataset # Circular imports. return sum( Dataset.objects(organization=org).visible().count() for org in self.organizations) @cached_property def followers_org_count(self): """Return the number of followers of user's organizations.""" from udata.models import Follow # Circular imports. return sum( Follow.objects(following=org).count() for org in self.organizations) @property def datasets_count(self): """Return the number of datasets of the user.""" return self.metrics.get('datasets', 0) @property def followers_count(self): """Return the number of followers of the user.""" return self.metrics.get('followers', 0) def generate_api_key(self): s = JSONWebSignatureSerializer(current_app.config['SECRET_KEY']) self.apikey = s.dumps({ 'user': str(self.id), 'time': time(), }) def clear_api_key(self): self.apikey = None @classmethod def get(cls, id_or_slug): obj = cls.objects(slug=id_or_slug).first() return obj or cls.objects.get_or_404(id=id_or_slug) @classmethod def pre_save(cls, sender, document, **kwargs): cls.before_save.send(document) @classmethod def post_save(cls, sender, document, **kwargs): cls.after_save.send(document) if kwargs.get('created'): cls.on_create.send(document) else: cls.on_update.send(document) @cached_property def json_ld(self): result = { '@type': 'Person', '@context': 'http://schema.org', 'name': self.fullname, } if self.about: result['description'] = mdstrip(self.about) if self.avatar_url: result['image'] = self.avatar_url if self.website: result['url'] = self.website return result def mark_as_deleted(self): copied_user = copy(self) self.email = '{}@deleted'.format(self.id) self.password = None self.active = False self.first_name = 'DELETED' self.last_name = 'DELETED' self.avatar = None self.avatar_url = None self.website = None self.about = None self.deleted = datetime.now() self.save() for organization in self.organizations: organization.members = [ member for member in organization.members if member.user != self ] organization.save() for discussion in Discussion.objects(discussion__posted_by=self): for message in discussion.discussion: if message.posted_by == self: message.content = 'DELETED' discussion.save() Follow.objects(follower=self).delete() Follow.objects(following=self).delete() mail.send(_('Account deletion'), copied_user, 'account_deleted')
class ResourceMixin(object): id = db.AutoUUIDField(primary_key=True) title = db.StringField(verbose_name="Title", required=True) description = db.StringField() filetype = db.StringField(choices=RESOURCE_TYPES.keys(), default='file', required=True) url = db.URLField(required=True) urlhash = db.StringField() checksum = db.EmbeddedDocumentField(Checksum) format = db.StringField() mime = db.StringField() filesize = db.IntField() # `size` is a reserved keyword for mongoengine. extras = db.ExtrasField() created_at = db.DateTimeField(default=datetime.now, required=True) modified = db.DateTimeField(default=datetime.now, required=True) published = db.DateTimeField(default=datetime.now, required=True) deleted = db.DateTimeField() def clean(self): super(ResourceMixin, self).clean() if not self.urlhash or 'url' in self._get_changed_fields(): self.urlhash = hash_url(self.url) @property def closed_format(self): """Return True if the specified format is in CLOSED_FORMATS.""" return self.format and self.format.lower() in CLOSED_FORMATS def check_availability(self): ''' Return the check status from extras if any. NB: `unknown` will evaluate to True in the aggregate checks using `all([])` (dataset, organization, user). ''' return self.extras.get('check:available', 'unknown') @property def latest(self): ''' Permanent link to the latest version of this resource. If this resource is updated and `url` changes, this property won't. ''' return url_for('datasets.resource', id=self.id, _external=True) @cached_property def json_ld(self): result = { '@type': 'DataDownload', '@id': str(self.id), 'url': self.latest, 'name': self.title or _('Nameless resource'), 'contentUrl': self.url, 'dateCreated': self.created_at.isoformat(), 'dateModified': self.modified.isoformat(), 'datePublished': self.published.isoformat(), 'extras': [get_json_ld_extra(*item) for item in self.extras.items()], } if 'views' in self.metrics: result['interactionStatistic'] = { '@type': 'InteractionCounter', 'interactionType': { '@type': 'DownloadAction', }, 'userInteractionCount': self.metrics['views'] } if self.format: result['encodingFormat'] = self.format if self.filesize: result['contentSize'] = self.filesize if self.mime: result['fileFormat'] = self.mime if self.description: result['description'] = mdstrip(self.description) # These 2 values are not standard if self.checksum: result['checksum'] = self.checksum.value, result['checksumType'] = self.checksum.type or 'sha1' return result
class NestedFake(db.EmbeddedDocument): key = db.StringField() value = db.IntField()
class User(db.Document, WithMetrics, UserMixin): slug = db.SlugField(max_length=255, required=True, populate_from='fullname') email = db.StringField(max_length=255, required=True) password = db.StringField() active = db.BooleanField() roles = db.ListField(db.ReferenceField(Role), default=[]) first_name = db.StringField(max_length=255, required=True) last_name = db.StringField(max_length=255, required=True) avatar_url = db.URLField() avatar = db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES) website = db.URLField() about = db.StringField() prefered_language = db.StringField() apikey = db.StringField() created_at = db.DateTimeField(default=datetime.now, required=True) confirmed_at = db.DateTimeField() last_login_at = db.DateTimeField() current_login_at = db.DateTimeField() last_login_ip = db.StringField() current_login_ip = db.StringField() login_count = db.IntField() deleted = db.DateTimeField() ext = db.MapField(db.GenericEmbeddedDocumentField()) extras = db.ExtrasField() before_save = Signal() after_save = Signal() on_create = Signal() on_update = Signal() before_delete = Signal() after_delete = Signal() on_delete = Signal() meta = { 'allow_inheritance': True, 'indexes': ['-created_at', 'slug', 'apikey'], 'ordering': ['-created_at'] } def __str__(self): return self.fullname __unicode__ = __str__ @property def fullname(self): return ' '.join((self.first_name or '', self.last_name or '')).strip() @cached_property def organizations(self): from udata.core.organization.models import Organization return Organization.objects(members__user=self) @property def sysadmin(self): return self.has_role('admin') def url_for(self, *args, **kwargs): return url_for('users.show', user=self, *args, **kwargs) display_url = property(url_for) @property def external_url(self): return self.url_for(_external=True) @property def visible(self): count = self.metrics.get('datasets', 0) + self.metrics.get('reuses', 0) return count > 0 @cached_property def resources_availability(self): """Return the percentage of availability for resources.""" # Flatten the list. availabilities = list( chain(*[org.check_availability() for org in self.organizations])) if availabilities: # Trick will work because it's a sum() of booleans. return round(100. * sum(availabilities) / len(availabilities), 2) else: return 0 @cached_property def datasets_org_count(self): """Return the number of datasets of user's organizations.""" from udata.models import Dataset # Circular imports. return sum( Dataset.objects(organization=org).visible().count() for org in self.organizations) @cached_property def followers_org_count(self): """Return the number of followers of user's organizations.""" from udata.models import FollowOrg # Circular imports. return sum( FollowOrg.objects(following=org).count() for org in self.organizations) @property def datasets_count(self): """Return the number of datasets of the user.""" return self.metrics.get('datasets', 0) @property def followers_count(self): """Return the number of followers of the user.""" return self.metrics.get('followers', 0) def generate_api_key(self): s = JSONWebSignatureSerializer(current_app.config['SECRET_KEY']) self.apikey = s.dumps({ 'user': str(self.id), 'time': time(), }) def clear_api_key(self): self.apikey = None @classmethod def get(cls, id_or_slug): obj = cls.objects(slug=id_or_slug).first() return obj or cls.objects.get_or_404(id=id_or_slug) @classmethod def pre_save(cls, sender, document, **kwargs): cls.before_save.send(document) @classmethod def post_save(cls, sender, document, **kwargs): cls.after_save.send(document) if kwargs.get('created'): cls.on_create.send(document) else: cls.on_update.send(document)
class GeoZone(db.Document): id = db.StringField(primary_key=True) slug = db.StringField(required=True) name = db.StringField(required=True) level = db.StringField(required=True) code = db.StringField(required=True) geom = db.MultiPolygonField() parents = db.ListField() keys = db.DictField() validity = db.EmbeddedDocumentField(db.DateRange) ancestors = db.ListField() successors = db.ListField() population = db.IntField() area = db.FloatField() wikipedia = db.StringField() dbpedia = db.StringField() flag = db.ImageField(fs=logos) blazon = db.ImageField(fs=logos) logo = db.ImageField(fs=logos) meta = { 'indexes': [ 'name', 'parents', ('level', 'code'), ], 'queryset_class': GeoZoneQuerySet } def __unicode__(self): return self.id __str__ = __unicode__ def __html__(self): """In use within the admin.""" return '{name} <i>({code})</i>'.format(name=gettext(self.name), code=self.code) def logo_url(self, external=False): flag_filename = self.flag.filename blazon_filename = self.blazon.filename if flag_filename and self.flag.fs.exists(flag_filename): return self.flag.fs.url(flag_filename, external=external) elif blazon_filename and self.blazon.fs.exists(blazon_filename): return self.blazon.fs.url(blazon_filename, external=external) else: return '' @property def keys_values(self): """Key values might be a list or not, always return a list.""" keys_values = [] for value in self.keys.values(): if isinstance(value, list): keys_values += value elif not str(value).startswith('-'): # Avoid -99. keys_values.append(value) return keys_values @cached_property def level_code(self): """Truncated level code for the sake of readability.""" # Either 'region', 'departement' or 'commune', # useful to match TERRITORY_DATASETS keys. return self.id.split(':')[1] @cached_property def level_name(self): """Truncated level name for the sake of readability.""" if self.level.startswith('fr:'): return self.level[3:] # Keep the whole level name as a fallback (e.g. `country:fr`) return self.level @cached_property def level_i18n_name(self): """In use within templates for dynamic translations.""" for level, name in spatial_granularities: if self.level == level: return name return self.level_name # Fallback that should never happen. @cached_property def ancestors_objects(self): """Ancestors objects sorted by name.""" ancestors_objects = [] for ancestor in self.ancestors: try: ancestor_object = GeoZone.objects.get(id=ancestor) except GeoZone.DoesNotExist: continue ancestors_objects.append(ancestor_object) ancestors_objects.sort(key=lambda a: a.name) return ancestors_objects @cached_property def child_level(self): """Return the child level given handled levels.""" HANDLED_LEVELS = current_app.config.get('HANDLED_LEVELS') try: return HANDLED_LEVELS[HANDLED_LEVELS.index(self.level) - 1] except (IndexError, ValueError): return None @cached_property def parent_level(self): """Return the parent level given handled levels.""" HANDLED_LEVELS = current_app.config.get('HANDLED_LEVELS') try: return HANDLED_LEVELS[HANDLED_LEVELS.index(self.level) + 1] except (IndexError, ValueError): return None @property def url(self): return url_for('territories.territory', territory=self) @property def external_url(self): return url_for('territories.territory', territory=self, _external=True) @cached_property def wikipedia_url(self): """Computed wikipedia URL from the DBpedia one.""" return (self.dbpedia.replace('dbpedia', 'wikipedia').replace('resource', 'wiki')) @cached_property def postal_string(self): """Return a list of postal codes separated by commas.""" return ', '.join(self.keys.get('postal', [])) @property def parents_objects(self): if self.parent_level: for parent in self.parents: if parent.startswith(self.parent_level): yield GeoZone.objects.get(id=parent, level=self.parent_level) @cached_property def current_parent(self): today = date.today() for parent in self.parents_objects: if parent.valid_at(today): return parent @property def children(self): return (GeoZone.objects(level=self.child_level, parents__in=[self.id]).order_by('name')) @property def biggest_children(self): return self.children.order_by('-population', '-area')[:10] @property def handled_level(self): return self.level in current_app.config.get('HANDLED_LEVELS') def valid_at(self, valid_date): if not self.validity: return True return self.validity.start <= valid_date <= self.validity.end def toGeoJSON(self): return { 'id': self.id, 'type': 'Feature', 'geometry': self.geom, 'properties': { 'slug': self.slug, 'name': gettext(self.name), 'level': self.level, 'code': self.code, 'validity': self.validity, 'parents': self.parents, 'keys': self.keys, 'population': self.population, 'area': self.area, 'logo': self.logo_url(external=True) } }
class Site(WithMetrics, db.Document): id = db.StringField(primary_key=True) title = db.StringField(required=True) keywords = db.ListField(db.StringField()) feed_size = db.IntField(required=True, default=DEFAULT_FEED_SIZE) configs = db.DictField() themes = db.DictField() settings = db.EmbeddedDocumentField(SiteSettings, default=SiteSettings) __metrics_keys__ = [ 'max_dataset_followers', 'max_dataset_reuses', 'max_reuse_datasets', 'max_reuse_followers', 'max_org_followers', 'max_org_reuses', 'max_org_datasets', 'datasets', 'discussions', 'followers', 'organizations', 'public-service', 'resources', 'reuses', 'users', ] def __str__(self): return self.title or '' def count_users(self): from udata.models import User self.metrics['users'] = User.objects(confirmed_at__ne=None, deleted=None).count() self.save() def count_org(self): from udata.models import Organization self.metrics['organizations'] = Organization.objects.visible().count() self.save() def count_org_for_badge(self, badge_kind): from udata.models import Organization self.metrics[badge_kind] = Organization.objects( badges__kind=badge_kind).count() self.save() def count_datasets(self): from udata.models import Dataset self.metrics['datasets'] = Dataset.objects.visible().count() self.save() def count_resources(self): self.metrics['resources'] = next( Dataset.objects.visible().aggregate( {'$project': { 'resources': 1 }}, {'$unwind': '$resources'}, {'$group': { '_id': 'result', 'count': { '$sum': 1 } }}), {}).get('count', 0) self.save() def count_reuses(self): self.metrics['reuses'] = Reuse.objects.visible().count() self.save() def count_followers(self): from udata.models import Follow self.metrics['followers'] = Follow.objects(until=None).count() self.save() def count_discussions(self): from udata.models import Discussion self.metrics['discussions'] = Discussion.objects.count() self.save() def count_max_dataset_followers(self): dataset = (Dataset.objects(metrics__followers__gt=0).visible(). order_by('-metrics.followers').first()) self.metrics['max_dataset_followers'] = dataset.metrics[ 'followers'] if dataset else 0 self.save() def count_max_dataset_reuses(self): dataset = (Dataset.objects(metrics__reuses__gt=0).visible().order_by( '-metrics.reuses').first()) self.metrics[ 'max_dataset_reuses'] = dataset.metrics['reuses'] if dataset else 0 self.save() def count_max_reuse_datasets(self): reuse = (Reuse.objects(metrics__datasets__gt=0).visible().order_by( '-metrics.datasets').first()) self.metrics[ 'max_reuse_datasets'] = reuse.metrics['datasets'] if reuse else 0 self.save() def count_max_reuse_followers(self): reuse = (Reuse.objects(metrics__followers__gt=0).visible().order_by( '-metrics.followers').first()) self.metrics[ 'max_reuse_followers'] = reuse.metrics['followers'] if reuse else 0 self.save() def count_max_org_followers(self): org = (Organization.objects(metrics__followers__gt=0).visible(). order_by('-metrics.followers').first()) self.metrics[ 'max_org_followers'] = org.metrics['followers'] if org else 0 self.save() def count_max_org_reuses(self): org = (Organization.objects(metrics__reuses__gt=0).visible().order_by( '-metrics.reuses').first()) self.metrics['max_org_reuses'] = org.metrics['reuses'] if org else 0 self.save() def count_max_org_datasets(self): org = (Organization.objects(metrics__datasets__gt=0).visible(). order_by('-metrics.datasets').first()) self.metrics[ 'max_org_datasets'] = org.metrics['datasets'] if org else 0 self.save()
class ResourceMixin(object): id = db.AutoUUIDField(primary_key=True) title = db.StringField(verbose_name="Title", required=True) description = db.StringField() filetype = db.StringField( choices=RESOURCE_FILETYPES.keys(), default='file', required=True) type = db.StringField( choices=RESOURCE_TYPES.keys(), default='main', required=True) url = db.URLField(required=True) urlhash = db.StringField() checksum = db.EmbeddedDocumentField(Checksum) format = db.StringField() mime = db.StringField() filesize = db.IntField() # `size` is a reserved keyword for mongoengine. extras = db.ExtrasField() created_at = db.DateTimeField(default=datetime.now, required=True) modified = db.DateTimeField(default=datetime.now, required=True) published = db.DateTimeField(default=datetime.now, required=True) deleted = db.DateTimeField() def clean(self): super(ResourceMixin, self).clean() if not self.urlhash or 'url' in self._get_changed_fields(): self.urlhash = hash_url(self.url) @cached_property # Accessed at least 2 times in front rendering def preview_url(self): return get_preview_url(self) @property def closed_or_no_format(self): """ Return True if the specified format is in CLOSED_FORMATS or no format has been specified. """ return not self.format or self.format.lower() in CLOSED_FORMATS def check_availability(self): ''' Return the check status from extras if any. NB: `unknown` will evaluate to True in the aggregate checks using `all([])` (dataset, organization, user). ''' return self.extras.get('check:available', 'unknown') def need_check(self): '''Does the resource needs to be checked against its linkchecker? We check unavailable resources often, unless they go over the threshold. Available resources are checked less and less frequently based on their historical availability. ''' min_cache_duration, max_cache_duration, ko_threshold = [ current_app.config.get(k) for k in ( 'LINKCHECKING_MIN_CACHE_DURATION', 'LINKCHECKING_MAX_CACHE_DURATION', 'LINKCHECKING_UNAVAILABLE_THRESHOLD', ) ] count_availability = self.extras.get('check:count-availability', 1) is_available = self.check_availability() if is_available == 'unknown': return True elif is_available or count_availability > ko_threshold: delta = min(min_cache_duration * count_availability, max_cache_duration) else: delta = min_cache_duration if self.extras.get('check:date'): limit_date = datetime.now() - timedelta(minutes=delta) check_date = self.extras['check:date'] if not isinstance(check_date, datetime): try: check_date = parse_dt(check_date) except (ValueError, TypeError): return True if check_date >= limit_date: return False return True @property def latest(self): ''' Permanent link to the latest version of this resource. If this resource is updated and `url` changes, this property won't. ''' return url_for('datasets.resource', id=self.id, _external=True) @cached_property def json_ld(self): result = { '@type': 'DataDownload', '@id': str(self.id), 'url': self.latest, 'name': self.title or _('Nameless resource'), 'contentUrl': self.url, 'dateCreated': self.created_at.isoformat(), 'dateModified': self.modified.isoformat(), 'datePublished': self.published.isoformat(), 'extras': [get_json_ld_extra(*item) for item in self.extras.items()], } if 'views' in self.metrics: result['interactionStatistic'] = { '@type': 'InteractionCounter', 'interactionType': { '@type': 'DownloadAction', }, 'userInteractionCount': self.metrics['views'] } if self.format: result['encodingFormat'] = self.format if self.filesize: result['contentSize'] = self.filesize if self.mime: result['fileFormat'] = self.mime if self.description: result['description'] = mdstrip(self.description) return result
class ResourceMixin(object): id = db.AutoUUIDField(primary_key=True) title = db.StringField(verbose_name="Title", required=True) description = db.StringField() filetype = db.StringField(choices=RESOURCE_TYPES.keys(), default='file', required=True) url = db.StringField() urlhash = db.StringField() checksum = db.EmbeddedDocumentField(Checksum) format = db.StringField() mime = db.StringField() filesize = db.IntField() # `size` is a reserved keyword for mongoengine. created_at = db.DateTimeField(default=datetime.now, required=True) modified = db.DateTimeField(default=datetime.now, required=True) published = db.DateTimeField(default=datetime.now, required=True) deleted = db.DateTimeField() def clean(self): super(ResourceMixin, self).clean() if not self.urlhash or 'url' in self._get_changed_fields(): self.urlhash = hash_url(self.url) @property def closed_format(self): """Return True if the specified format is in CLOSED_FORMATS.""" return self.format.lower() in CLOSED_FORMATS def check_availability(self, group): """Check if a resource is reachable against a Croquemort server. Return a boolean. """ if self.filetype == 'remote': # We perform a quick check for performances matters. error, response = check_url_from_cache(self.url, group) if error or 'status' not in response: return False elif int(response['status']) >= httplib.INTERNAL_SERVER_ERROR: return False else: return True else: return True # We consider that API cases (types) are OK. @property def is_available(self): return self.check_availability(group=None) @property def latest(self): ''' Permanent link to the latest version of this resource. If this resource is updated and `url` changes, this property won't. ''' return url_for('datasets.resource', id=self.id, _external=True) @cached_property def json_ld(self): result = { '@type': 'DataDownload', '@id': str(self.id), 'url': self.latest, 'name': self.title or _('Nameless resource'), 'contentUrl': self.url, 'dateCreated': self.created_at.isoformat(), 'dateModified': self.modified.isoformat(), 'datePublished': self.published.isoformat(), } if 'views' in self.metrics: result['interactionStatistic'] = { '@type': 'InteractionCounter', 'interactionType': { '@type': 'DownloadAction', }, 'userInteractionCount': self.metrics['views'] } if self.format: result['encodingFormat'] = self.format if self.filesize: result['contentSize'] = self.filesize if self.mime: result['fileFormat'] = self.mime if self.description: result['description'] = mdstrip(self.description) # These 2 values are not standard if self.checksum: result['checksum'] = self.checksum.value, result['checksumType'] = self.checksum.type or 'sha1' return result