class FakeActivity(Activity): key = 'fake' related_to = db.ReferenceField(FakeModel)
class Message(db.EmbeddedDocument): content = db.StringField(required=True) posted_on = db.DateTimeField(default=datetime.now, required=True) posted_by = db.ReferenceField('User')
class Dataset(WithMetrics, BadgeMixin, db.Owned, db.Document): created_at = DateTimeField(verbose_name=_('Creation date'), default=datetime.now, required=True) last_modified = DateTimeField(verbose_name=_('Last modification date'), default=datetime.now, required=True) title = db.StringField(required=True) acronym = db.StringField(max_length=128) # /!\ do not set directly the slug when creating or updating a dataset # this will break the search indexation slug = db.SlugField(max_length=255, required=True, populate_from='title', update=True, follow=True) description = db.StringField(required=True, default='') license = db.ReferenceField('License') tags = db.TagListField() resources = db.ListField(db.EmbeddedDocumentField(Resource)) private = db.BooleanField(default=False) frequency = db.StringField(choices=list(UPDATE_FREQUENCIES.keys())) frequency_date = db.DateTimeField(verbose_name=_('Future date of update')) temporal_coverage = db.EmbeddedDocumentField(db.DateRange) spatial = db.EmbeddedDocumentField(SpatialCoverage) ext = db.MapField(db.GenericEmbeddedDocumentField()) extras = db.ExtrasField() featured = db.BooleanField(required=True, default=False) deleted = db.DateTimeField() archived = db.DateTimeField() def __str__(self): return self.title or '' __badges__ = { PIVOTAL_DATA: _('Pivotal data'), } __search_metrics__ = Object(properties={ 'reuses': Integer(), 'followers': Integer(), 'views': Integer(), }) __metrics_keys__ = [ 'discussions', 'reuses', 'followers', 'views', ] meta = { 'indexes': [ '-created_at', 'slug', 'resources.id', 'resources.urlhash', ] + db.Owned.meta['indexes'], 'ordering': ['-created_at'], 'queryset_class': DatasetQuerySet, } before_save = signal('Dataset.before_save') after_save = signal('Dataset.after_save') on_create = signal('Dataset.on_create') on_update = signal('Dataset.on_update') before_delete = signal('Dataset.before_delete') after_delete = signal('Dataset.after_delete') on_delete = signal('Dataset.on_delete') on_archive = signal('Dataset.on_archive') on_resource_added = signal('Dataset.on_resource_added') verbose_name = _('dataset') @classmethod def pre_save(cls, sender, document, **kwargs): cls.before_save.send(document) @classmethod def post_save(cls, sender, document, **kwargs): if 'post_save' in kwargs.get('ignores', []): return cls.after_save.send(document) if kwargs.get('created'): cls.on_create.send(document) else: cls.on_update.send(document) if document.deleted: cls.on_delete.send(document) if document.archived: cls.on_archive.send(document) if kwargs.get('resource_added'): cls.on_resource_added.send(document, resource_id=kwargs['resource_added']) def clean(self): super(Dataset, self).clean() if self.frequency in LEGACY_FREQUENCIES: self.frequency = LEGACY_FREQUENCIES[self.frequency] def url_for(self, *args, **kwargs): return endpoint_for('datasets.show', 'api.dataset', dataset=self, *args, **kwargs) display_url = property(url_for) @property def is_visible(self): return not self.is_hidden @property def is_hidden(self): return (len(self.resources) == 0 or self.private or self.deleted or self.archived) @property def full_title(self): if not self.acronym: return self.title return '{title} ({acronym})'.format(**self._data) @property def external_url(self): return self.url_for(_external=True) @property def image_url(self): if self.organization: return self.organization.logo.url elif self.owner: return self.owner.avatar.url @property def frequency_label(self): return UPDATE_FREQUENCIES.get(self.frequency or 'unknown', UPDATE_FREQUENCIES['unknown']) def check_availability(self): """Check if resources from that dataset are available. Return a list of (boolean or 'unknown') """ # Only check remote resources. remote_resources = [ resource for resource in self.resources if resource.filetype == 'remote' ] if not remote_resources: return [] return [resource.check_availability() for resource in remote_resources] @property def last_update(self): if self.resources: return max(resource.published for resource in self.resources) else: return self.last_modified @property def next_update(self): """Compute the next expected update date, given the frequency and last_update. Return None if the frequency is not handled. """ delta = None if self.frequency == 'daily': delta = timedelta(days=1) elif self.frequency == 'weekly': delta = timedelta(weeks=1) elif self.frequency == 'fortnighly': delta = timedelta(weeks=2) elif self.frequency == 'monthly': delta = timedelta(weeks=4) elif self.frequency == 'bimonthly': delta = timedelta(weeks=4 * 2) elif self.frequency == 'quarterly': delta = timedelta(weeks=52 / 4) elif self.frequency == 'biannual': delta = timedelta(weeks=52 / 2) elif self.frequency == 'annual': delta = timedelta(weeks=52) elif self.frequency == 'biennial': delta = timedelta(weeks=52 * 2) elif self.frequency == 'triennial': delta = timedelta(weeks=52 * 3) elif self.frequency == 'quinquennial': delta = timedelta(weeks=52 * 5) if delta is None: return else: return self.last_update + delta @cached_property def quality(self): """Return a dict filled with metrics related to the inner quality of the dataset: * number of tags * description length * and so on """ from udata.models import Discussion # noqa: Prevent circular imports result = {} if not self.id: # Quality is only relevant on saved Datasets return result if self.frequency != 'unknown': result['frequency'] = self.frequency if self.next_update: result['update_in'] = -(self.next_update - datetime.now()).days if self.tags: result['tags_count'] = len(self.tags) if self.description: result['description_length'] = len(self.description) if self.resources: result['has_resources'] = True result['has_only_closed_or_no_formats'] = all( resource.closed_or_no_format for resource in self.resources) result['has_unavailable_resources'] = not all( self.check_availability()) discussions = Discussion.objects(subject=self) if discussions: result['discussions'] = len(discussions) result['has_untreated_discussions'] = not all( discussion.person_involved(self.owner) for discussion in discussions) result['score'] = self.compute_quality_score(result) return result def compute_quality_score(self, quality): """Compute the score related to the quality of that dataset.""" score = 0 UNIT = 2 if 'update_in' in quality: # TODO: should be related to frequency. if quality['update_in'] < 0: score += UNIT else: score -= UNIT if 'tags_count' in quality: if quality['tags_count'] > 3: score += UNIT if 'description_length' in quality: if quality['description_length'] > 100: score += UNIT if 'has_resources' in quality: if quality['has_only_closed_or_no_formats']: score -= UNIT else: score += UNIT if quality['has_unavailable_resources']: score -= UNIT else: score += UNIT if 'discussions' in quality: if quality['has_untreated_discussions']: score -= UNIT else: score += UNIT if score < 0: return 0 return score @classmethod def get(cls, id_or_slug): obj = cls.objects(slug=id_or_slug).first() return obj or cls.objects.get_or_404(id=id_or_slug) def add_resource(self, resource): '''Perform an atomic prepend for a new resource''' resource.validate() self.update( __raw__={ '$push': { 'resources': { '$each': [resource.to_mongo()], '$position': 0 } } }) self.reload() post_save.send(self.__class__, document=self, resource_added=resource.id) def update_resource(self, resource): '''Perform an atomic update for an existing resource''' index = self.resources.index(resource) data = {'resources__{index}'.format(index=index): resource} self.update(**data) self.reload() post_save.send(self.__class__, document=self) def remove_resource(self, resource): # Deletes resource's file from file storage if resource.fs_filename is not None: storages.resources.delete(resource.fs_filename) self.resources.remove(resource) @property def community_resources(self): return self.id and CommunityResource.objects.filter(dataset=self) or [] @cached_property def json_ld(self): result = { '@context': 'http://schema.org', '@type': 'Dataset', '@id': str(self.id), 'alternateName': self.slug, 'dateCreated': self.created_at.isoformat(), 'dateModified': self.last_modified.isoformat(), 'url': endpoint_for('datasets.show', 'api.dataset', dataset=self, _external=True), 'name': self.title, 'keywords': ','.join(self.tags), 'distribution': [resource.json_ld for resource in self.resources], # Theses values are not standard 'contributedDistribution': [resource.json_ld for resource in self.community_resources], 'extras': [get_json_ld_extra(*item) for item in self.extras.items()], } if self.description: result['description'] = mdstrip(self.description) if self.license and self.license.url: result['license'] = self.license.url if self.organization: author = self.organization.json_ld elif self.owner: author = self.owner.json_ld else: author = None if author: result['author'] = author return result @property def views_count(self): return self.metrics.get('views', 0) def count_discussions(self): from udata.models import Discussion self.metrics['discussions'] = Discussion.objects(subject=self, closed=None).count() self.save() def count_reuses(self): from udata.models import Reuse self.metrics['reuses'] = Reuse.objects(datasets=self).visible().count() self.save() def count_followers(self): from udata.models import Follow self.metrics['followers'] = Follow.objects( until=None).followers(self).count() self.save()
class FollowUser(Follow): following = db.ReferenceField(User)
class GeoLevel(db.Document): id = db.StringField(primary_key=True) name = db.StringField(required=True) parents = db.ListField(db.ReferenceField('self'))
class Fake(db.Document): name = db.StringField() nested = db.ListField(db.ReferenceField(Nested))
class DatasetIssue(Issue): subject = db.ReferenceField(Dataset)
class UserFollowedUser(FollowActivity, Activity): key = 'user:followed' label = _('followed a user') related_to = db.ReferenceField(User) template = 'activity/user.html'
class User(db.Document, WithMetrics, UserMixin): slug = db.SlugField(max_length=255, required=True, populate_from='fullname') email = db.StringField(max_length=255, required=True) password = db.StringField() active = db.BooleanField() roles = db.ListField(db.ReferenceField(Role), default=[]) first_name = db.StringField(max_length=255, required=True) last_name = db.StringField(max_length=255, required=True) avatar_url = db.URLField() avatar = db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES) website = db.URLField() about = db.StringField() prefered_language = db.StringField() apikey = db.StringField() created_at = db.DateTimeField(default=datetime.now, required=True) confirmed_at = db.DateTimeField() last_login_at = db.DateTimeField() current_login_at = db.DateTimeField() last_login_ip = db.StringField() current_login_ip = db.StringField() login_count = db.IntField() deleted = db.DateTimeField() ext = db.MapField(db.GenericEmbeddedDocumentField()) extras = db.ExtrasField() before_save = Signal() after_save = Signal() on_create = Signal() on_update = Signal() before_delete = Signal() after_delete = Signal() on_delete = Signal() meta = { 'allow_inheritance': True, 'indexes': ['-created_at', 'slug', 'apikey'], 'ordering': ['-created_at'] } def __str__(self): return self.fullname __unicode__ = __str__ @property def fullname(self): return ' '.join((self.first_name or '', self.last_name or '')).strip() @cached_property def organizations(self): from udata.core.organization.models import Organization return Organization.objects(members__user=self) @property def sysadmin(self): return self.has_role('admin') def url_for(self, *args, **kwargs): return url_for('users.show', user=self, *args, **kwargs) display_url = property(url_for) @property def external_url(self): return self.url_for(_external=True) @property def visible(self): count = self.metrics.get('datasets', 0) + self.metrics.get('reuses', 0) return count > 0 @cached_property def resources_availability(self): """Return the percentage of availability for resources.""" # Flatten the list. availabilities = list( chain(*[org.check_availability() for org in self.organizations])) if availabilities: # Trick will work because it's a sum() of booleans. return round(100. * sum(availabilities) / len(availabilities), 2) else: return 0 @cached_property def datasets_org_count(self): """Return the number of datasets of user's organizations.""" from udata.models import Dataset # Circular imports. return sum( Dataset.objects(organization=org).visible().count() for org in self.organizations) @cached_property def followers_org_count(self): """Return the number of followers of user's organizations.""" from udata.models import FollowOrg # Circular imports. return sum( FollowOrg.objects(following=org).count() for org in self.organizations) @property def datasets_count(self): """Return the number of datasets of the user.""" return self.metrics.get('datasets', 0) @property def followers_count(self): """Return the number of followers of the user.""" return self.metrics.get('followers', 0) def generate_api_key(self): s = JSONWebSignatureSerializer(current_app.config['SECRET_KEY']) self.apikey = s.dumps({ 'user': str(self.id), 'time': time(), }) def clear_api_key(self): self.apikey = None @classmethod def get(cls, id_or_slug): obj = cls.objects(slug=id_or_slug).first() return obj or cls.objects.get_or_404(id=id_or_slug) @classmethod def pre_save(cls, sender, document, **kwargs): cls.before_save.send(document) @classmethod def post_save(cls, sender, document, **kwargs): cls.after_save.send(document) if kwargs.get('created'): cls.on_create.send(document) else: cls.on_update.send(document)
class DatasetDiscussion(Discussion): subject = db.ReferenceField(Dataset)
class OAuth2Client(ClientMixin, db.Datetimed, db.Document): secret = db.StringField(default=lambda: gen_salt(50)) name = db.StringField(required=True) description = db.StringField() owner = db.ReferenceField('User') organization = db.ReferenceField('Organization') image = db.ImageField(fs=images, basename=default_image_basename, thumbnails=[150, 25]) redirect_uris = db.ListField(db.StringField()) scope = db.StringField(default='default') grant_types = db.ListField(db.StringField()) response_types = db.ListField(db.StringField()) confidential = db.BooleanField(default=False) internal = db.BooleanField(default=False) meta = {'collection': 'oauth2_client'} def get_client_id(self): return str(self.id) @property def client_id(self): return self.get_client_id() @property def client_secret(self): return self.secret @property def default_redirect_uri(self): return self.redirect_uris[0] def get_default_redirect_uri(self): return self.default_redirect_uri def get_allowed_scope(self, scope): if not scope: return '' allowed = set(scope_to_list(self.scope)) return list_to_scope([s for s in scope.split() if s in allowed]) def check_redirect_uri(self, redirect_uri): return redirect_uri in self.redirect_uris def check_client_secret(self, client_secret): return self.secret == client_secret def check_token_endpoint_auth_method(self, method): if not self.has_client_secret(): return method == 'none' return method in ('client_secret_post', 'client_secret_basic') def check_response_type(self, response_type): return True def check_grant_type(self, grant_type): return True def check_requested_scope(self, scope): allowed = set(self.scope) return allowed.issuperset(set(scope)) def has_client_secret(self): return bool(self.secret)
class Dataset(WithMetrics, BadgeMixin, db.Datetimed, db.Document): title = db.StringField(max_length=255, required=True) slug = db.SlugField(max_length=255, required=True, populate_from='title', update=True) description = db.StringField(required=True, default='') license = db.ReferenceField('License') tags = db.ListField(db.StringField()) resources = db.ListField(db.EmbeddedDocumentField(Resource)) private = db.BooleanField() owner = db.ReferenceField('User', reverse_delete_rule=db.NULLIFY) organization = db.ReferenceField('Organization', reverse_delete_rule=db.NULLIFY) frequency = db.StringField(choices=UPDATE_FREQUENCIES.keys()) frequency_date = db.DateTimeField(verbose_name=_('Future date of update')) temporal_coverage = db.EmbeddedDocumentField(db.DateRange) spatial = db.EmbeddedDocumentField(SpatialCoverage) ext = db.MapField(db.GenericEmbeddedDocumentField()) extras = db.ExtrasField() featured = db.BooleanField(required=True, default=False) deleted = db.DateTimeField() def __str__(self): return self.title or '' __unicode__ = __str__ __badges__ = { PIVOTAL_DATA: _('Pivotal data'), } meta = { 'allow_inheritance': True, 'indexes': [ '-created_at', 'slug', 'organization', 'resources.id', 'resources.urlhash', ], 'ordering': ['-created_at'], 'queryset_class': DatasetQuerySet, } before_save = signal('Dataset.before_save') after_save = signal('Dataset.after_save') on_create = signal('Dataset.on_create') on_update = signal('Dataset.on_update') before_delete = signal('Dataset.before_delete') after_delete = signal('Dataset.after_delete') on_delete = signal('Dataset.on_delete') verbose_name = _('dataset') @classmethod def pre_save(cls, sender, document, **kwargs): cls.before_save.send(document) @classmethod def post_save(cls, sender, document, **kwargs): cls.after_save.send(document) if kwargs.get('created'): cls.on_create.send(document) else: cls.on_update.send(document) def url_for(self, *args, **kwargs): return url_for('datasets.show', dataset=self, *args, **kwargs) display_url = property(url_for) @property def external_url(self): return self.url_for(_external=True) @property def image_url(self): if self.organization: return self.organization.logo.url elif self.owner: return self.owner.avatar.url @property def frequency_label(self): return UPDATE_FREQUENCIES.get(self.frequency or 'unknown', UPDATE_FREQUENCIES['unknown']) def check_availability(self): """Check if resources from that dataset are available. Return a list of booleans. """ # Only check remote resources. remote_resources = [ resource for resource in self.resources if resource.filetype == 'remote' ] if not remote_resources: return [] # First, we try to retrieve all data from the group (slug). error, response = check_url_from_group(self.slug) if error: # The group is unknown, the check will be performed by resource. return [ resource.check_availability(self.slug) for resource in remote_resources ] else: return [ int(url_infos['status']) == 200 for url_infos in response['urls'] ] @property def last_update(self): if self.resources: return max(resource.published for resource in self.resources) else: return self.last_modified @property def next_update(self): """Compute the next expected update date, given the frequency and last_update. Return None if the frequency is not handled. """ delta = None if self.frequency == 'daily': delta = timedelta(days=1) elif self.frequency == 'weekly': delta = timedelta(weeks=1) elif self.frequency == 'fortnighly': delta = timedelta(weeks=2) elif self.frequency == 'monthly': delta = timedelta(weeks=4) elif self.frequency == 'bimonthly': delta = timedelta(weeks=4 * 2) elif self.frequency == 'quarterly': delta = timedelta(weeks=52 / 4) elif self.frequency == 'biannual': delta = timedelta(weeks=52 / 2) elif self.frequency == 'annual': delta = timedelta(weeks=52) elif self.frequency == 'biennial': delta = timedelta(weeks=52 * 2) elif self.frequency == 'triennial': delta = timedelta(weeks=52 * 3) elif self.frequency == 'quinquennial': delta = timedelta(weeks=52 * 5) if delta is None: return else: return self.last_update + delta @cached_property def quality(self): """Return a dict filled with metrics related to the inner quality of the dataset: * number of tags * description length * and so on """ result = {} if self.next_update: result['frequency'] = self.frequency result['update_in'] = -(self.next_update - datetime.now()).days if self.tags: result['tags_count'] = len(self.tags) if self.description: result['description_length'] = len(self.description) if self.resources: result['has_resources'] = True result['has_only_closed_formats'] = all( resource.closed_format for resource in self.resources) result['has_unavailable_resources'] = not all( self.check_availability()) discussions = DatasetDiscussion.objects(subject=self.id) if discussions: result['discussions'] = len(discussions) result['has_untreated_discussions'] = not all( discussion.person_involved(self.owner) for discussion in discussions) result['score'] = self.compute_quality_score(result) return result def compute_quality_score(self, quality): """Compute the score related to the quality of that dataset.""" score = 0 UNIT = 2 if 'frequency' in quality: # TODO: should be related to frequency. if quality['update_in'] < 0: score += UNIT else: score -= UNIT if 'tags_count' in quality: if quality['tags_count'] > 3: score += UNIT if 'description_length' in quality: if quality['description_length'] > 100: score += UNIT if 'has_resources' in quality: if quality['has_only_closed_formats']: score -= UNIT else: score += UNIT if quality['has_unavailable_resources']: score -= UNIT else: score += UNIT if 'discussions' in quality: if quality['has_untreated_discussions']: score -= UNIT else: score += UNIT if score < 0: return 0 return score @classmethod def get(cls, id_or_slug): obj = cls.objects(slug=id_or_slug).first() return obj or cls.objects.get_or_404(id=id_or_slug) def add_resource(self, resource): '''Perform an atomic prepend for a new resource''' self.update( __raw__={ '$push': { 'resources': { '$each': [resource.to_mongo()], '$position': 0 } } }) self.reload() post_save.send(self.__class__, document=self) def update_resource(self, resource): '''Perform an atomic update for an existing resource''' index = self.resources.index(resource) data = {'resources__{index}'.format(index=index): resource} self.update(**data) self.reload() post_save.send(self.__class__, document=self) @property def community_resources(self): return self.id and CommunityResource.objects.filter(dataset=self) or []
class DatasetRelatedActivity(object): template = 'activity/dataset.html' related_to = db.ReferenceField('Dataset')
class ReuseRelatedActivity(object): template = 'activity/reuse.html' related_to = db.ReferenceField('Reuse')
class Reuse(db.Datetimed, WithMetrics, db.Document): title = db.StringField(max_length=255, required=True) slug = db.SlugField(max_length=255, required=True, populate_from='title', update=True) description = db.StringField(required=True) type = db.StringField(required=True, choices=REUSE_TYPES.keys()) url = db.StringField(required=True) urlhash = db.StringField(required=True, unique=True) image_url = db.StringField() image = db.ImageField(fs=images, basename=default_image_basename, max_size=IMAGE_MAX_SIZE, thumbnails=IMAGE_SIZES) datasets = db.ListField( db.ReferenceField('Dataset', reverse_delete_rule=db.PULL)) tags = db.ListField(db.StringField()) private = db.BooleanField() owner = db.ReferenceField('User', reverse_delete_rule=db.NULLIFY) organization = db.ReferenceField('Organization', reverse_delete_rule=db.NULLIFY) ext = db.MapField(db.GenericEmbeddedDocumentField()) extras = db.ExtrasField() featured = db.BooleanField() deleted = db.DateTimeField() def __str__(self): return self.title or '' __unicode__ = __str__ meta = { 'allow_inheritance': True, 'indexes': ['-created_at', 'owner', 'urlhash'], 'ordering': ['-created_at'], 'queryset_class': ReuseQuerySet, } before_save = Signal() after_save = Signal() on_create = Signal() on_update = Signal() before_delete = Signal() after_delete = Signal() on_delete = Signal() verbose_name = _('reuse') @classmethod def pre_save(cls, sender, document, **kwargs): # Emit before_save cls.before_save.send(document) @classmethod def post_save(cls, sender, document, **kwargs): cls.after_save.send(document) if kwargs.get('created'): cls.on_create.send(document) else: cls.on_update.send(document) @property def display_url(self): return url_for('reuses.show', reuse=self) @property def external_url(self): return url_for('reuses.show', reuse=self, _external=True) @property def type_label(self): return REUSE_TYPES[self.type] def clean(self): '''Auto populate urlhash from url''' if not self.urlhash or 'url' in self._get_changed_fields(): self.urlhash = hash_url(self.url) super(Reuse, self).clean() @classmethod def get(cls, id_or_slug): obj = cls.objects(slug=id_or_slug).first() return obj or cls.objects.get_or_404(id=id_or_slug) @classmethod def url_exists(cls, url): urlhash = hash_url(url) return cls.objects(urlhash=urlhash).count() > 0
class ModelExplicit(db.Document): name = db.StringField() target = db.ReferenceField(Target)
class FakeIssue(Issue): subject = db.ReferenceField(Fake)
class Reuse(db.Datetimed, WithMetrics, BadgeMixin, db.Owned, db.Document): title = db.StringField(required=True) slug = db.SlugField(max_length=255, required=True, populate_from='title', update=True) description = db.StringField(required=True) type = db.StringField(required=True, choices=REUSE_TYPES.keys()) url = db.StringField(required=True) urlhash = db.StringField(required=True, unique=True) image_url = db.StringField() image = db.ImageField(fs=images, basename=default_image_basename, max_size=IMAGE_MAX_SIZE, thumbnails=IMAGE_SIZES) datasets = db.ListField( db.ReferenceField('Dataset', reverse_delete_rule=db.PULL)) tags = db.TagListField() # badges = db.ListField(db.EmbeddedDocumentField(ReuseBadge)) private = db.BooleanField() ext = db.MapField(db.GenericEmbeddedDocumentField()) extras = db.ExtrasField() featured = db.BooleanField() deleted = db.DateTimeField() def __str__(self): return self.title or '' __unicode__ = __str__ __badges__ = {} meta = { 'indexes': ['-created_at', 'urlhash'] + db.Owned.meta['indexes'], 'ordering': ['-created_at'], 'queryset_class': ReuseQuerySet, } before_save = Signal() after_save = Signal() on_create = Signal() on_update = Signal() before_delete = Signal() after_delete = Signal() on_delete = Signal() verbose_name = _('reuse') @classmethod def pre_save(cls, sender, document, **kwargs): # Emit before_save cls.before_save.send(document) @classmethod def post_save(cls, sender, document, **kwargs): cls.after_save.send(document) if kwargs.get('created'): cls.on_create.send(document) else: cls.on_update.send(document) if document.deleted: cls.on_delete.send(document) def url_for(self, *args, **kwargs): return url_for('reuses.show', reuse=self, *args, **kwargs) display_url = property(url_for) @property def external_url(self): return self.url_for(_external=True) @property def type_label(self): return REUSE_TYPES[self.type] def clean(self): '''Auto populate urlhash from url''' if not self.urlhash or 'url' in self._get_changed_fields(): self.urlhash = hash_url(self.url) super(Reuse, self).clean() @classmethod def get(cls, id_or_slug): obj = cls.objects(slug=id_or_slug).first() return obj or cls.objects.get_or_404(id=id_or_slug) @classmethod def url_exists(cls, url): urlhash = hash_url(url) return cls.objects(urlhash=urlhash).count() > 0 @cached_property def json_ld(self): result = { '@context': 'http://schema.org', '@type': 'CreativeWork', 'alternateName': self.slug, 'dateCreated': self.created_at.isoformat(), 'dateModified': self.last_modified.isoformat(), 'url': url_for('reuses.show', reuse=self, _external=True), 'name': self.title, 'isBasedOnUrl': self.url, } if self.description: result['description'] = mdstrip(self.description) if self.organization: author = self.organization.json_ld elif self.owner: author = self.owner.json_ld else: author = None if author: result['author'] = author return result
class Dataset(WithMetrics, db.Datetimed, db.Document): title = db.StringField(max_length=255, required=True) slug = db.SlugField(max_length=255, required=True, populate_from='title', update=True) description = db.StringField(required=True, default='') license = db.ReferenceField('License') tags = db.ListField(db.StringField()) resources = db.ListField(db.EmbeddedDocumentField(Resource)) community_resources = db.ListField(db.EmbeddedDocumentField(Resource)) private = db.BooleanField() owner = db.ReferenceField('User', reverse_delete_rule=db.NULLIFY) organization = db.ReferenceField('Organization', reverse_delete_rule=db.NULLIFY) supplier = db.ReferenceField('Organization', reverse_delete_rule=db.NULLIFY) frequency = db.StringField(choices=UPDATE_FREQUENCIES.keys()) temporal_coverage = db.EmbeddedDocumentField(db.DateRange) spatial = db.EmbeddedDocumentField(SpatialCoverage) ext = db.MapField(db.GenericEmbeddedDocumentField()) extras = db.ExtrasField() featured = db.BooleanField(required=True, default=False) deleted = db.DateTimeField() def __str__(self): return self.title or '' __unicode__ = __str__ meta = { 'allow_inheritance': True, 'indexes': [ '-created_at', 'slug', 'organization', 'supplier', 'resources.id', 'resources.urlhash', ], 'ordering': ['-created_at'], 'queryset_class': DatasetQuerySet, } before_save = Signal() after_save = Signal() on_create = Signal() on_update = Signal() before_delete = Signal() after_delete = Signal() on_delete = Signal() verbose_name = _('dataset') @classmethod def pre_save(cls, sender, document, **kwargs): cls.before_save.send(document) @classmethod def post_save(cls, sender, document, **kwargs): cls.after_save.send(document) if kwargs.get('created'): cls.on_create.send(document) else: cls.on_update.send(document) @property def display_url(self): return url_for('datasets.show', dataset=self) @property def external_url(self): return url_for('datasets.show', dataset=self, _external=True) @property def image_url(self): if self.organization: return self.organization.logo.url elif self.owner: return self.owner.avatar.url @property def frequency_label(self): return UPDATE_FREQUENCIES.get(self.frequency or 'unknown', UPDATE_FREQUENCIES['unknown']) @classmethod def get(cls, id_or_slug): obj = cls.objects(slug=id_or_slug).first() return obj or cls.objects.get_or_404(id=id_or_slug) def add_resource(self, resource): '''Perform an atomic prepend for a new resource''' self.update( __raw__={ '$push': { 'resources': { '$each': [resource.to_mongo()], '$position': 0 } } }) self.reload() def add_community_resource(self, resource): '''Perform an atomic prepend for a new resource''' self.update( __raw__={ '$push': { 'community_resources': { '$each': [resource.to_mongo()], '$position': 0 } } }) self.reload()
class OrgRelatedActivity(object): related_to = db.ReferenceField('Organization') template = 'activity/organization.html'
class FollowDataset(Follow): following = db.ReferenceField(Dataset)
class FollowOrg(Follow): following = db.ReferenceField(Organization)
class User(db.Document, WithMetrics, UserMixin): slug = db.SlugField(max_length=255, required=True, populate_from='fullname') email = db.StringField(max_length=255, required=True) password = db.StringField() active = db.BooleanField() roles = db.ListField(db.ReferenceField(Role), default=[]) first_name = db.StringField(max_length=255, required=True) last_name = db.StringField(max_length=255, required=True) avatar_url = db.URLField() avatar = db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES) website = db.URLField() about = db.StringField() prefered_language = db.StringField() apikey = db.StringField() created_at = db.DateTimeField(default=datetime.now, required=True) confirmed_at = db.DateTimeField() last_login_at = db.DateTimeField() current_login_at = db.DateTimeField() last_login_ip = db.StringField() current_login_ip = db.StringField() login_count = db.IntField() deleted = db.DateTimeField() ext = db.MapField(db.GenericEmbeddedDocumentField()) extras = db.ExtrasField() before_save = Signal() after_save = Signal() on_create = Signal() on_update = Signal() before_delete = Signal() after_delete = Signal() on_delete = Signal() meta = { 'allow_inheritance': True, 'indexes': ['-created_at', 'slug', 'apikey'], 'ordering': ['-created_at'] } def __str__(self): return self.fullname __unicode__ = __str__ @property def fullname(self): return ' '.join((self.first_name or '', self.last_name or '')).strip() @cached_property def organizations(self): from udata.core.organization.models import Organization return Organization.objects(members__user=self) @property def sysadmin(self): return bool(getattr(g, 'sysadmin')) @property def display_url(self): return url_for('users.show', user=self) @property def visible(self): return self.metrics.get('datasets', 0) + self.metrics.get('reuses', 0) > 0 def generate_api_key(self): s = JSONWebSignatureSerializer(current_app.config['SECRET_KEY']) self.apikey = s.dumps({ 'user': str(self.id), 'time': time(), }) def clear_api_key(self): self.apikey = None @classmethod def get(cls, id_or_slug): obj = cls.objects(slug=id_or_slug).first() return obj or cls.objects.get_or_404(id=id_or_slug) @classmethod def pre_save(cls, sender, document, **kwargs): cls.before_save.send(document) @classmethod def post_save(cls, sender, document, **kwargs): cls.after_save.send(document) if kwargs.get('created'): cls.on_create.send(document) else: cls.on_update.send(document)
class Ownable(db.Document): owner = db.ReferenceField(User)
class Ownable(db.Document): owner = db.ReferenceField(User) organization = db.ReferenceField(Organization)
class ReuseIssue(Issue): subject = db.ReferenceField(Reuse)
class User(WithMetrics, UserMixin, db.Document): slug = db.SlugField(max_length=255, required=True, populate_from='fullname') email = db.StringField(max_length=255, required=True, unique=True) password = db.StringField() active = db.BooleanField() roles = db.ListField(db.ReferenceField(Role), default=[]) first_name = db.StringField(max_length=255, required=True) last_name = db.StringField(max_length=255, required=True) avatar_url = db.URLField() avatar = db.ImageField(fs=avatars, basename=default_image_basename, thumbnails=AVATAR_SIZES) website = db.URLField() about = db.StringField() prefered_language = db.StringField() apikey = db.StringField() created_at = db.DateTimeField(default=datetime.now, required=True) # The field below is required for Flask-security # when SECURITY_CONFIRMABLE is True confirmed_at = db.DateTimeField() password_rotation_demanded = db.DateTimeField() password_rotation_performed = db.DateTimeField() # The 5 fields below are required for Flask-security # when SECURITY_TRACKABLE is True last_login_at = db.DateTimeField() current_login_at = db.DateTimeField() last_login_ip = db.StringField() current_login_ip = db.StringField() login_count = db.IntField() deleted = db.DateTimeField() ext = db.MapField(db.GenericEmbeddedDocumentField()) extras = db.ExtrasField() before_save = Signal() after_save = Signal() on_create = Signal() on_update = Signal() before_delete = Signal() after_delete = Signal() on_delete = Signal() meta = { 'indexes': ['-created_at', 'slug', 'apikey'], 'ordering': ['-created_at'] } __search_metrics__ = Object( properties={ 'datasets': Integer(), 'reuses': Integer(), 'followers': Integer(), 'views': Integer() }) __metrics_keys__ = [ 'datasets', 'reuses', 'following', 'followers', ] def __str__(self): return self.fullname @property def fullname(self): return ' '.join((self.first_name or '', self.last_name or '')).strip() @cached_property def organizations(self): from udata.core.organization.models import Organization return Organization.objects(members__user=self, deleted__exists=False) @property def sysadmin(self): return self.has_role('admin') def url_for(self, *args, **kwargs): return url_for('users.show', user=self, *args, **kwargs) display_url = property(url_for) @property def external_url(self): return self.url_for(_external=True) @property def visible(self): count = self.metrics.get('datasets', 0) + self.metrics.get('reuses', 0) return count > 0 and self.active @cached_property def resources_availability(self): """Return the percentage of availability for resources.""" # Flatten the list. availabilities = list( chain(*[org.check_availability() for org in self.organizations])) # Filter out the unknown availabilities = [a for a in availabilities if type(a) is bool] if availabilities: # Trick will work because it's a sum() of booleans. return round(100. * sum(availabilities) / len(availabilities), 2) # if nothing is unavailable, everything is considered OK return 100 @cached_property def datasets_org_count(self): """Return the number of datasets of user's organizations.""" from udata.models import Dataset # Circular imports. return sum( Dataset.objects(organization=org).visible().count() for org in self.organizations) @cached_property def followers_org_count(self): """Return the number of followers of user's organizations.""" from udata.models import Follow # Circular imports. return sum( Follow.objects(following=org).count() for org in self.organizations) @property def datasets_count(self): """Return the number of datasets of the user.""" return self.metrics.get('datasets', 0) @property def followers_count(self): """Return the number of followers of the user.""" return self.metrics.get('followers', 0) def generate_api_key(self): s = JSONWebSignatureSerializer(current_app.config['SECRET_KEY']) byte_str = s.dumps({ 'user': str(self.id), 'time': time(), }) self.apikey = byte_str.decode() def clear_api_key(self): self.apikey = None @classmethod def get(cls, id_or_slug): obj = cls.objects(slug=id_or_slug).first() return obj or cls.objects.get_or_404(id=id_or_slug) @classmethod def pre_save(cls, sender, document, **kwargs): cls.before_save.send(document) @classmethod def post_save(cls, sender, document, **kwargs): cls.after_save.send(document) if kwargs.get('created'): cls.on_create.send(document) else: cls.on_update.send(document) @cached_property def json_ld(self): result = { '@type': 'Person', '@context': 'http://schema.org', 'name': self.fullname, } if self.about: result['description'] = mdstrip(self.about) if self.avatar_url: result['image'] = self.avatar_url if self.website: result['url'] = self.website return result def mark_as_deleted(self): copied_user = copy(self) self.email = '{}@deleted'.format(self.id) self.slug = 'deleted' self.password = None self.active = False self.first_name = 'DELETED' self.last_name = 'DELETED' self.avatar = None self.avatar_url = None self.website = None self.about = None self.extras = None self.apikey = None self.deleted = datetime.now() self.save() for organization in self.organizations: organization.members = [ member for member in organization.members if member.user != self ] organization.save() for discussion in Discussion.objects(discussion__posted_by=self): for message in discussion.discussion: if message.posted_by == self: message.content = 'DELETED' discussion.save() Follow.objects(follower=self).delete() Follow.objects(following=self).delete() mail.send(_('Account deletion'), copied_user, 'account_deleted') def count_datasets(self): from udata.models import Dataset self.metrics['datasets'] = Dataset.objects( owner=self).visible().count() self.save() def count_reuses(self): from udata.models import Reuse self.metrics['reuses'] = Reuse.objects(owner=self).visible().count() self.save() def count_followers(self): from udata.models import Follow self.metrics['followers'] = Follow.objects( until=None).followers(self).count() self.save() def count_following(self): from udata.models import Follow self.metrics['following'] = Follow.objects.following(self).count() self.save()
class FollowReuse(Follow): following = db.ReferenceField(Reuse)
class FollowFake(Follow): following = db.ReferenceField(Fake)
class Team(db.EmbeddedDocument): name = db.StringField(required=True) slug = db.SlugField(max_length=255, required=True, populate_from='name', update=True, unique=False) description = db.StringField() members = db.ListField(db.ReferenceField('User'))