class FakeActivity(Activity):
key = 'fake'
related_to = db.ReferenceField(FakeModel)
class Message(db.EmbeddedDocument):
content = db.StringField(required=True)
posted_on = db.DateTimeField(default=datetime.now, required=True)
posted_by = db.ReferenceField('User')
class Dataset(WithMetrics, BadgeMixin, db.Owned, db.Document):
created_at = DateTimeField(verbose_name=_('Creation date'),
default=datetime.now,
required=True)
last_modified = DateTimeField(verbose_name=_('Last modification date'),
default=datetime.now,
required=True)
title = db.StringField(required=True)
acronym = db.StringField(max_length=128)
# /!\ do not set directly the slug when creating or updating a dataset
# this will break the search indexation
slug = db.SlugField(max_length=255,
required=True,
populate_from='title',
update=True,
follow=True)
description = db.StringField(required=True, default='')
license = db.ReferenceField('License')
tags = db.TagListField()
resources = db.ListField(db.EmbeddedDocumentField(Resource))
private = db.BooleanField(default=False)
frequency = db.StringField(choices=list(UPDATE_FREQUENCIES.keys()))
frequency_date = db.DateTimeField(verbose_name=_('Future date of update'))
temporal_coverage = db.EmbeddedDocumentField(db.DateRange)
spatial = db.EmbeddedDocumentField(SpatialCoverage)
ext = db.MapField(db.GenericEmbeddedDocumentField())
extras = db.ExtrasField()
featured = db.BooleanField(required=True, default=False)
deleted = db.DateTimeField()
archived = db.DateTimeField()
def __str__(self):
return self.title or ''
__badges__ = {
PIVOTAL_DATA: _('Pivotal data'),
}
__search_metrics__ = Object(properties={
'reuses': Integer(),
'followers': Integer(),
'views': Integer(),
})
__metrics_keys__ = [
'discussions',
'reuses',
'followers',
'views',
]
meta = {
'indexes': [
'-created_at',
'slug',
'resources.id',
'resources.urlhash',
] + db.Owned.meta['indexes'],
'ordering': ['-created_at'],
'queryset_class':
DatasetQuerySet,
}
before_save = signal('Dataset.before_save')
after_save = signal('Dataset.after_save')
on_create = signal('Dataset.on_create')
on_update = signal('Dataset.on_update')
before_delete = signal('Dataset.before_delete')
after_delete = signal('Dataset.after_delete')
on_delete = signal('Dataset.on_delete')
on_archive = signal('Dataset.on_archive')
on_resource_added = signal('Dataset.on_resource_added')
verbose_name = _('dataset')
@classmethod
def pre_save(cls, sender, document, **kwargs):
cls.before_save.send(document)
@classmethod
def post_save(cls, sender, document, **kwargs):
if 'post_save' in kwargs.get('ignores', []):
return
cls.after_save.send(document)
if kwargs.get('created'):
cls.on_create.send(document)
else:
cls.on_update.send(document)
if document.deleted:
cls.on_delete.send(document)
if document.archived:
cls.on_archive.send(document)
if kwargs.get('resource_added'):
cls.on_resource_added.send(document,
resource_id=kwargs['resource_added'])
def clean(self):
super(Dataset, self).clean()
if self.frequency in LEGACY_FREQUENCIES:
self.frequency = LEGACY_FREQUENCIES[self.frequency]
def url_for(self, *args, **kwargs):
return endpoint_for('datasets.show',
'api.dataset',
dataset=self,
*args,
**kwargs)
display_url = property(url_for)
@property
def is_visible(self):
return not self.is_hidden
@property
def is_hidden(self):
return (len(self.resources) == 0 or self.private or self.deleted
or self.archived)
@property
def full_title(self):
if not self.acronym:
return self.title
return '{title} ({acronym})'.format(**self._data)
@property
def external_url(self):
return self.url_for(_external=True)
@property
def image_url(self):
if self.organization:
return self.organization.logo.url
elif self.owner:
return self.owner.avatar.url
@property
def frequency_label(self):
return UPDATE_FREQUENCIES.get(self.frequency or 'unknown',
UPDATE_FREQUENCIES['unknown'])
def check_availability(self):
"""Check if resources from that dataset are available.
Return a list of (boolean or 'unknown')
"""
# Only check remote resources.
remote_resources = [
resource for resource in self.resources
if resource.filetype == 'remote'
]
if not remote_resources:
return []
return [resource.check_availability() for resource in remote_resources]
@property
def last_update(self):
if self.resources:
return max(resource.published for resource in self.resources)
else:
return self.last_modified
@property
def next_update(self):
"""Compute the next expected update date,
given the frequency and last_update.
Return None if the frequency is not handled.
"""
delta = None
if self.frequency == 'daily':
delta = timedelta(days=1)
elif self.frequency == 'weekly':
delta = timedelta(weeks=1)
elif self.frequency == 'fortnighly':
delta = timedelta(weeks=2)
elif self.frequency == 'monthly':
delta = timedelta(weeks=4)
elif self.frequency == 'bimonthly':
delta = timedelta(weeks=4 * 2)
elif self.frequency == 'quarterly':
delta = timedelta(weeks=52 / 4)
elif self.frequency == 'biannual':
delta = timedelta(weeks=52 / 2)
elif self.frequency == 'annual':
delta = timedelta(weeks=52)
elif self.frequency == 'biennial':
delta = timedelta(weeks=52 * 2)
elif self.frequency == 'triennial':
delta = timedelta(weeks=52 * 3)
elif self.frequency == 'quinquennial':
delta = timedelta(weeks=52 * 5)
if delta is None:
return
else:
return self.last_update + delta
@cached_property
def quality(self):
"""Return a dict filled with metrics related to the inner
quality of the dataset:
* number of tags
* description length
* and so on
"""
from udata.models import Discussion # noqa: Prevent circular imports
result = {}
if not self.id:
# Quality is only relevant on saved Datasets
return result
if self.frequency != 'unknown':
result['frequency'] = self.frequency
if self.next_update:
result['update_in'] = -(self.next_update - datetime.now()).days
if self.tags:
result['tags_count'] = len(self.tags)
if self.description:
result['description_length'] = len(self.description)
if self.resources:
result['has_resources'] = True
result['has_only_closed_or_no_formats'] = all(
resource.closed_or_no_format for resource in self.resources)
result['has_unavailable_resources'] = not all(
self.check_availability())
discussions = Discussion.objects(subject=self)
if discussions:
result['discussions'] = len(discussions)
result['has_untreated_discussions'] = not all(
discussion.person_involved(self.owner)
for discussion in discussions)
result['score'] = self.compute_quality_score(result)
return result
def compute_quality_score(self, quality):
"""Compute the score related to the quality of that dataset."""
score = 0
UNIT = 2
if 'update_in' in quality:
# TODO: should be related to frequency.
if quality['update_in'] < 0:
score += UNIT
else:
score -= UNIT
if 'tags_count' in quality:
if quality['tags_count'] > 3:
score += UNIT
if 'description_length' in quality:
if quality['description_length'] > 100:
score += UNIT
if 'has_resources' in quality:
if quality['has_only_closed_or_no_formats']:
score -= UNIT
else:
score += UNIT
if quality['has_unavailable_resources']:
score -= UNIT
else:
score += UNIT
if 'discussions' in quality:
if quality['has_untreated_discussions']:
score -= UNIT
else:
score += UNIT
if score < 0:
return 0
return score
@classmethod
def get(cls, id_or_slug):
obj = cls.objects(slug=id_or_slug).first()
return obj or cls.objects.get_or_404(id=id_or_slug)
def add_resource(self, resource):
'''Perform an atomic prepend for a new resource'''
resource.validate()
self.update(
__raw__={
'$push': {
'resources': {
'$each': [resource.to_mongo()],
'$position': 0
}
}
})
self.reload()
post_save.send(self.__class__,
document=self,
resource_added=resource.id)
def update_resource(self, resource):
'''Perform an atomic update for an existing resource'''
index = self.resources.index(resource)
data = {'resources__{index}'.format(index=index): resource}
self.update(**data)
self.reload()
post_save.send(self.__class__, document=self)
def remove_resource(self, resource):
# Deletes resource's file from file storage
if resource.fs_filename is not None:
storages.resources.delete(resource.fs_filename)
self.resources.remove(resource)
@property
def community_resources(self):
return self.id and CommunityResource.objects.filter(dataset=self) or []
@cached_property
def json_ld(self):
result = {
'@context':
'http://schema.org',
'@type':
'Dataset',
'@id':
str(self.id),
'alternateName':
self.slug,
'dateCreated':
self.created_at.isoformat(),
'dateModified':
self.last_modified.isoformat(),
'url':
endpoint_for('datasets.show',
'api.dataset',
dataset=self,
_external=True),
'name':
self.title,
'keywords':
','.join(self.tags),
'distribution': [resource.json_ld for resource in self.resources],
# Theses values are not standard
'contributedDistribution':
[resource.json_ld for resource in self.community_resources],
'extras':
[get_json_ld_extra(*item) for item in self.extras.items()],
}
if self.description:
result['description'] = mdstrip(self.description)
if self.license and self.license.url:
result['license'] = self.license.url
if self.organization:
author = self.organization.json_ld
elif self.owner:
author = self.owner.json_ld
else:
author = None
if author:
result['author'] = author
return result
@property
def views_count(self):
return self.metrics.get('views', 0)
def count_discussions(self):
from udata.models import Discussion
self.metrics['discussions'] = Discussion.objects(subject=self,
closed=None).count()
self.save()
def count_reuses(self):
from udata.models import Reuse
self.metrics['reuses'] = Reuse.objects(datasets=self).visible().count()
self.save()
def count_followers(self):
from udata.models import Follow
self.metrics['followers'] = Follow.objects(
until=None).followers(self).count()
self.save()
class FollowUser(Follow):
following = db.ReferenceField(User)
class GeoLevel(db.Document):
id = db.StringField(primary_key=True)
name = db.StringField(required=True)
parents = db.ListField(db.ReferenceField('self'))
class Fake(db.Document):
name = db.StringField()
nested = db.ListField(db.ReferenceField(Nested))
class DatasetIssue(Issue):
subject = db.ReferenceField(Dataset)
class UserFollowedUser(FollowActivity, Activity):
key = 'user:followed'
label = _('followed a user')
related_to = db.ReferenceField(User)
template = 'activity/user.html'
class User(db.Document, WithMetrics, UserMixin):
slug = db.SlugField(max_length=255,
required=True,
populate_from='fullname')
email = db.StringField(max_length=255, required=True)
password = db.StringField()
active = db.BooleanField()
roles = db.ListField(db.ReferenceField(Role), default=[])
first_name = db.StringField(max_length=255, required=True)
last_name = db.StringField(max_length=255, required=True)
avatar_url = db.URLField()
avatar = db.ImageField(fs=avatars,
basename=default_image_basename,
thumbnails=AVATAR_SIZES)
website = db.URLField()
about = db.StringField()
prefered_language = db.StringField()
apikey = db.StringField()
created_at = db.DateTimeField(default=datetime.now, required=True)
confirmed_at = db.DateTimeField()
last_login_at = db.DateTimeField()
current_login_at = db.DateTimeField()
last_login_ip = db.StringField()
current_login_ip = db.StringField()
login_count = db.IntField()
deleted = db.DateTimeField()
ext = db.MapField(db.GenericEmbeddedDocumentField())
extras = db.ExtrasField()
before_save = Signal()
after_save = Signal()
on_create = Signal()
on_update = Signal()
before_delete = Signal()
after_delete = Signal()
on_delete = Signal()
meta = {
'allow_inheritance': True,
'indexes': ['-created_at', 'slug', 'apikey'],
'ordering': ['-created_at']
}
def __str__(self):
return self.fullname
__unicode__ = __str__
@property
def fullname(self):
return ' '.join((self.first_name or '', self.last_name or '')).strip()
@cached_property
def organizations(self):
from udata.core.organization.models import Organization
return Organization.objects(members__user=self)
@property
def sysadmin(self):
return self.has_role('admin')
def url_for(self, *args, **kwargs):
return url_for('users.show', user=self, *args, **kwargs)
display_url = property(url_for)
@property
def external_url(self):
return self.url_for(_external=True)
@property
def visible(self):
count = self.metrics.get('datasets', 0) + self.metrics.get('reuses', 0)
return count > 0
@cached_property
def resources_availability(self):
"""Return the percentage of availability for resources."""
# Flatten the list.
availabilities = list(
chain(*[org.check_availability() for org in self.organizations]))
if availabilities:
# Trick will work because it's a sum() of booleans.
return round(100. * sum(availabilities) / len(availabilities), 2)
else:
return 0
@cached_property
def datasets_org_count(self):
"""Return the number of datasets of user's organizations."""
from udata.models import Dataset # Circular imports.
return sum(
Dataset.objects(organization=org).visible().count()
for org in self.organizations)
@cached_property
def followers_org_count(self):
"""Return the number of followers of user's organizations."""
from udata.models import FollowOrg # Circular imports.
return sum(
FollowOrg.objects(following=org).count()
for org in self.organizations)
@property
def datasets_count(self):
"""Return the number of datasets of the user."""
return self.metrics.get('datasets', 0)
@property
def followers_count(self):
"""Return the number of followers of the user."""
return self.metrics.get('followers', 0)
def generate_api_key(self):
s = JSONWebSignatureSerializer(current_app.config['SECRET_KEY'])
self.apikey = s.dumps({
'user': str(self.id),
'time': time(),
})
def clear_api_key(self):
self.apikey = None
@classmethod
def get(cls, id_or_slug):
obj = cls.objects(slug=id_or_slug).first()
return obj or cls.objects.get_or_404(id=id_or_slug)
@classmethod
def pre_save(cls, sender, document, **kwargs):
cls.before_save.send(document)
@classmethod
def post_save(cls, sender, document, **kwargs):
cls.after_save.send(document)
if kwargs.get('created'):
cls.on_create.send(document)
else:
cls.on_update.send(document)
class DatasetDiscussion(Discussion):
subject = db.ReferenceField(Dataset)
class OAuth2Client(ClientMixin, db.Datetimed, db.Document):
secret = db.StringField(default=lambda: gen_salt(50))
name = db.StringField(required=True)
description = db.StringField()
owner = db.ReferenceField('User')
organization = db.ReferenceField('Organization')
image = db.ImageField(fs=images,
basename=default_image_basename,
thumbnails=[150, 25])
redirect_uris = db.ListField(db.StringField())
scope = db.StringField(default='default')
grant_types = db.ListField(db.StringField())
response_types = db.ListField(db.StringField())
confidential = db.BooleanField(default=False)
internal = db.BooleanField(default=False)
meta = {'collection': 'oauth2_client'}
def get_client_id(self):
return str(self.id)
@property
def client_id(self):
return self.get_client_id()
@property
def client_secret(self):
return self.secret
@property
def default_redirect_uri(self):
return self.redirect_uris[0]
def get_default_redirect_uri(self):
return self.default_redirect_uri
def get_allowed_scope(self, scope):
if not scope:
return ''
allowed = set(scope_to_list(self.scope))
return list_to_scope([s for s in scope.split() if s in allowed])
def check_redirect_uri(self, redirect_uri):
return redirect_uri in self.redirect_uris
def check_client_secret(self, client_secret):
return self.secret == client_secret
def check_token_endpoint_auth_method(self, method):
if not self.has_client_secret():
return method == 'none'
return method in ('client_secret_post', 'client_secret_basic')
def check_response_type(self, response_type):
return True
def check_grant_type(self, grant_type):
return True
def check_requested_scope(self, scope):
allowed = set(self.scope)
return allowed.issuperset(set(scope))
def has_client_secret(self):
return bool(self.secret)
class Dataset(WithMetrics, BadgeMixin, db.Datetimed, db.Document):
title = db.StringField(max_length=255, required=True)
slug = db.SlugField(max_length=255,
required=True,
populate_from='title',
update=True)
description = db.StringField(required=True, default='')
license = db.ReferenceField('License')
tags = db.ListField(db.StringField())
resources = db.ListField(db.EmbeddedDocumentField(Resource))
private = db.BooleanField()
owner = db.ReferenceField('User', reverse_delete_rule=db.NULLIFY)
organization = db.ReferenceField('Organization',
reverse_delete_rule=db.NULLIFY)
frequency = db.StringField(choices=UPDATE_FREQUENCIES.keys())
frequency_date = db.DateTimeField(verbose_name=_('Future date of update'))
temporal_coverage = db.EmbeddedDocumentField(db.DateRange)
spatial = db.EmbeddedDocumentField(SpatialCoverage)
ext = db.MapField(db.GenericEmbeddedDocumentField())
extras = db.ExtrasField()
featured = db.BooleanField(required=True, default=False)
deleted = db.DateTimeField()
def __str__(self):
return self.title or ''
__unicode__ = __str__
__badges__ = {
PIVOTAL_DATA: _('Pivotal data'),
}
meta = {
'allow_inheritance':
True,
'indexes': [
'-created_at',
'slug',
'organization',
'resources.id',
'resources.urlhash',
],
'ordering': ['-created_at'],
'queryset_class':
DatasetQuerySet,
}
before_save = signal('Dataset.before_save')
after_save = signal('Dataset.after_save')
on_create = signal('Dataset.on_create')
on_update = signal('Dataset.on_update')
before_delete = signal('Dataset.before_delete')
after_delete = signal('Dataset.after_delete')
on_delete = signal('Dataset.on_delete')
verbose_name = _('dataset')
@classmethod
def pre_save(cls, sender, document, **kwargs):
cls.before_save.send(document)
@classmethod
def post_save(cls, sender, document, **kwargs):
cls.after_save.send(document)
if kwargs.get('created'):
cls.on_create.send(document)
else:
cls.on_update.send(document)
def url_for(self, *args, **kwargs):
return url_for('datasets.show', dataset=self, *args, **kwargs)
display_url = property(url_for)
@property
def external_url(self):
return self.url_for(_external=True)
@property
def image_url(self):
if self.organization:
return self.organization.logo.url
elif self.owner:
return self.owner.avatar.url
@property
def frequency_label(self):
return UPDATE_FREQUENCIES.get(self.frequency or 'unknown',
UPDATE_FREQUENCIES['unknown'])
def check_availability(self):
"""Check if resources from that dataset are available.
Return a list of booleans.
"""
# Only check remote resources.
remote_resources = [
resource for resource in self.resources
if resource.filetype == 'remote'
]
if not remote_resources:
return []
# First, we try to retrieve all data from the group (slug).
error, response = check_url_from_group(self.slug)
if error:
# The group is unknown, the check will be performed by resource.
return [
resource.check_availability(self.slug)
for resource in remote_resources
]
else:
return [
int(url_infos['status']) == 200
for url_infos in response['urls']
]
@property
def last_update(self):
if self.resources:
return max(resource.published for resource in self.resources)
else:
return self.last_modified
@property
def next_update(self):
"""Compute the next expected update date,
given the frequency and last_update.
Return None if the frequency is not handled.
"""
delta = None
if self.frequency == 'daily':
delta = timedelta(days=1)
elif self.frequency == 'weekly':
delta = timedelta(weeks=1)
elif self.frequency == 'fortnighly':
delta = timedelta(weeks=2)
elif self.frequency == 'monthly':
delta = timedelta(weeks=4)
elif self.frequency == 'bimonthly':
delta = timedelta(weeks=4 * 2)
elif self.frequency == 'quarterly':
delta = timedelta(weeks=52 / 4)
elif self.frequency == 'biannual':
delta = timedelta(weeks=52 / 2)
elif self.frequency == 'annual':
delta = timedelta(weeks=52)
elif self.frequency == 'biennial':
delta = timedelta(weeks=52 * 2)
elif self.frequency == 'triennial':
delta = timedelta(weeks=52 * 3)
elif self.frequency == 'quinquennial':
delta = timedelta(weeks=52 * 5)
if delta is None:
return
else:
return self.last_update + delta
@cached_property
def quality(self):
"""Return a dict filled with metrics related to the inner
quality of the dataset:
* number of tags
* description length
* and so on
"""
result = {}
if self.next_update:
result['frequency'] = self.frequency
result['update_in'] = -(self.next_update - datetime.now()).days
if self.tags:
result['tags_count'] = len(self.tags)
if self.description:
result['description_length'] = len(self.description)
if self.resources:
result['has_resources'] = True
result['has_only_closed_formats'] = all(
resource.closed_format for resource in self.resources)
result['has_unavailable_resources'] = not all(
self.check_availability())
discussions = DatasetDiscussion.objects(subject=self.id)
if discussions:
result['discussions'] = len(discussions)
result['has_untreated_discussions'] = not all(
discussion.person_involved(self.owner)
for discussion in discussions)
result['score'] = self.compute_quality_score(result)
return result
def compute_quality_score(self, quality):
"""Compute the score related to the quality of that dataset."""
score = 0
UNIT = 2
if 'frequency' in quality:
# TODO: should be related to frequency.
if quality['update_in'] < 0:
score += UNIT
else:
score -= UNIT
if 'tags_count' in quality:
if quality['tags_count'] > 3:
score += UNIT
if 'description_length' in quality:
if quality['description_length'] > 100:
score += UNIT
if 'has_resources' in quality:
if quality['has_only_closed_formats']:
score -= UNIT
else:
score += UNIT
if quality['has_unavailable_resources']:
score -= UNIT
else:
score += UNIT
if 'discussions' in quality:
if quality['has_untreated_discussions']:
score -= UNIT
else:
score += UNIT
if score < 0:
return 0
return score
@classmethod
def get(cls, id_or_slug):
obj = cls.objects(slug=id_or_slug).first()
return obj or cls.objects.get_or_404(id=id_or_slug)
def add_resource(self, resource):
'''Perform an atomic prepend for a new resource'''
self.update(
__raw__={
'$push': {
'resources': {
'$each': [resource.to_mongo()],
'$position': 0
}
}
})
self.reload()
post_save.send(self.__class__, document=self)
def update_resource(self, resource):
'''Perform an atomic update for an existing resource'''
index = self.resources.index(resource)
data = {'resources__{index}'.format(index=index): resource}
self.update(**data)
self.reload()
post_save.send(self.__class__, document=self)
@property
def community_resources(self):
return self.id and CommunityResource.objects.filter(dataset=self) or []
class DatasetRelatedActivity(object):
template = 'activity/dataset.html'
related_to = db.ReferenceField('Dataset')
class ReuseRelatedActivity(object):
template = 'activity/reuse.html'
related_to = db.ReferenceField('Reuse')
class Reuse(db.Datetimed, WithMetrics, db.Document):
title = db.StringField(max_length=255, required=True)
slug = db.SlugField(max_length=255,
required=True,
populate_from='title',
update=True)
description = db.StringField(required=True)
type = db.StringField(required=True, choices=REUSE_TYPES.keys())
url = db.StringField(required=True)
urlhash = db.StringField(required=True, unique=True)
image_url = db.StringField()
image = db.ImageField(fs=images,
basename=default_image_basename,
max_size=IMAGE_MAX_SIZE,
thumbnails=IMAGE_SIZES)
datasets = db.ListField(
db.ReferenceField('Dataset', reverse_delete_rule=db.PULL))
tags = db.ListField(db.StringField())
private = db.BooleanField()
owner = db.ReferenceField('User', reverse_delete_rule=db.NULLIFY)
organization = db.ReferenceField('Organization',
reverse_delete_rule=db.NULLIFY)
ext = db.MapField(db.GenericEmbeddedDocumentField())
extras = db.ExtrasField()
featured = db.BooleanField()
deleted = db.DateTimeField()
def __str__(self):
return self.title or ''
__unicode__ = __str__
meta = {
'allow_inheritance': True,
'indexes': ['-created_at', 'owner', 'urlhash'],
'ordering': ['-created_at'],
'queryset_class': ReuseQuerySet,
}
before_save = Signal()
after_save = Signal()
on_create = Signal()
on_update = Signal()
before_delete = Signal()
after_delete = Signal()
on_delete = Signal()
verbose_name = _('reuse')
@classmethod
def pre_save(cls, sender, document, **kwargs):
# Emit before_save
cls.before_save.send(document)
@classmethod
def post_save(cls, sender, document, **kwargs):
cls.after_save.send(document)
if kwargs.get('created'):
cls.on_create.send(document)
else:
cls.on_update.send(document)
@property
def display_url(self):
return url_for('reuses.show', reuse=self)
@property
def external_url(self):
return url_for('reuses.show', reuse=self, _external=True)
@property
def type_label(self):
return REUSE_TYPES[self.type]
def clean(self):
'''Auto populate urlhash from url'''
if not self.urlhash or 'url' in self._get_changed_fields():
self.urlhash = hash_url(self.url)
super(Reuse, self).clean()
@classmethod
def get(cls, id_or_slug):
obj = cls.objects(slug=id_or_slug).first()
return obj or cls.objects.get_or_404(id=id_or_slug)
@classmethod
def url_exists(cls, url):
urlhash = hash_url(url)
return cls.objects(urlhash=urlhash).count() > 0
class ModelExplicit(db.Document):
name = db.StringField()
target = db.ReferenceField(Target)
class FakeIssue(Issue):
subject = db.ReferenceField(Fake)
class Reuse(db.Datetimed, WithMetrics, BadgeMixin, db.Owned, db.Document):
title = db.StringField(required=True)
slug = db.SlugField(max_length=255,
required=True,
populate_from='title',
update=True)
description = db.StringField(required=True)
type = db.StringField(required=True, choices=REUSE_TYPES.keys())
url = db.StringField(required=True)
urlhash = db.StringField(required=True, unique=True)
image_url = db.StringField()
image = db.ImageField(fs=images,
basename=default_image_basename,
max_size=IMAGE_MAX_SIZE,
thumbnails=IMAGE_SIZES)
datasets = db.ListField(
db.ReferenceField('Dataset', reverse_delete_rule=db.PULL))
tags = db.TagListField()
# badges = db.ListField(db.EmbeddedDocumentField(ReuseBadge))
private = db.BooleanField()
ext = db.MapField(db.GenericEmbeddedDocumentField())
extras = db.ExtrasField()
featured = db.BooleanField()
deleted = db.DateTimeField()
def __str__(self):
return self.title or ''
__unicode__ = __str__
__badges__ = {}
meta = {
'indexes': ['-created_at', 'urlhash'] + db.Owned.meta['indexes'],
'ordering': ['-created_at'],
'queryset_class': ReuseQuerySet,
}
before_save = Signal()
after_save = Signal()
on_create = Signal()
on_update = Signal()
before_delete = Signal()
after_delete = Signal()
on_delete = Signal()
verbose_name = _('reuse')
@classmethod
def pre_save(cls, sender, document, **kwargs):
# Emit before_save
cls.before_save.send(document)
@classmethod
def post_save(cls, sender, document, **kwargs):
cls.after_save.send(document)
if kwargs.get('created'):
cls.on_create.send(document)
else:
cls.on_update.send(document)
if document.deleted:
cls.on_delete.send(document)
def url_for(self, *args, **kwargs):
return url_for('reuses.show', reuse=self, *args, **kwargs)
display_url = property(url_for)
@property
def external_url(self):
return self.url_for(_external=True)
@property
def type_label(self):
return REUSE_TYPES[self.type]
def clean(self):
'''Auto populate urlhash from url'''
if not self.urlhash or 'url' in self._get_changed_fields():
self.urlhash = hash_url(self.url)
super(Reuse, self).clean()
@classmethod
def get(cls, id_or_slug):
obj = cls.objects(slug=id_or_slug).first()
return obj or cls.objects.get_or_404(id=id_or_slug)
@classmethod
def url_exists(cls, url):
urlhash = hash_url(url)
return cls.objects(urlhash=urlhash).count() > 0
@cached_property
def json_ld(self):
result = {
'@context': 'http://schema.org',
'@type': 'CreativeWork',
'alternateName': self.slug,
'dateCreated': self.created_at.isoformat(),
'dateModified': self.last_modified.isoformat(),
'url': url_for('reuses.show', reuse=self, _external=True),
'name': self.title,
'isBasedOnUrl': self.url,
}
if self.description:
result['description'] = mdstrip(self.description)
if self.organization:
author = self.organization.json_ld
elif self.owner:
author = self.owner.json_ld
else:
author = None
if author:
result['author'] = author
return result
class Dataset(WithMetrics, db.Datetimed, db.Document):
title = db.StringField(max_length=255, required=True)
slug = db.SlugField(max_length=255,
required=True,
populate_from='title',
update=True)
description = db.StringField(required=True, default='')
license = db.ReferenceField('License')
tags = db.ListField(db.StringField())
resources = db.ListField(db.EmbeddedDocumentField(Resource))
community_resources = db.ListField(db.EmbeddedDocumentField(Resource))
private = db.BooleanField()
owner = db.ReferenceField('User', reverse_delete_rule=db.NULLIFY)
organization = db.ReferenceField('Organization',
reverse_delete_rule=db.NULLIFY)
supplier = db.ReferenceField('Organization',
reverse_delete_rule=db.NULLIFY)
frequency = db.StringField(choices=UPDATE_FREQUENCIES.keys())
temporal_coverage = db.EmbeddedDocumentField(db.DateRange)
spatial = db.EmbeddedDocumentField(SpatialCoverage)
ext = db.MapField(db.GenericEmbeddedDocumentField())
extras = db.ExtrasField()
featured = db.BooleanField(required=True, default=False)
deleted = db.DateTimeField()
def __str__(self):
return self.title or ''
__unicode__ = __str__
meta = {
'allow_inheritance':
True,
'indexes': [
'-created_at',
'slug',
'organization',
'supplier',
'resources.id',
'resources.urlhash',
],
'ordering': ['-created_at'],
'queryset_class':
DatasetQuerySet,
}
before_save = Signal()
after_save = Signal()
on_create = Signal()
on_update = Signal()
before_delete = Signal()
after_delete = Signal()
on_delete = Signal()
verbose_name = _('dataset')
@classmethod
def pre_save(cls, sender, document, **kwargs):
cls.before_save.send(document)
@classmethod
def post_save(cls, sender, document, **kwargs):
cls.after_save.send(document)
if kwargs.get('created'):
cls.on_create.send(document)
else:
cls.on_update.send(document)
@property
def display_url(self):
return url_for('datasets.show', dataset=self)
@property
def external_url(self):
return url_for('datasets.show', dataset=self, _external=True)
@property
def image_url(self):
if self.organization:
return self.organization.logo.url
elif self.owner:
return self.owner.avatar.url
@property
def frequency_label(self):
return UPDATE_FREQUENCIES.get(self.frequency or 'unknown',
UPDATE_FREQUENCIES['unknown'])
@classmethod
def get(cls, id_or_slug):
obj = cls.objects(slug=id_or_slug).first()
return obj or cls.objects.get_or_404(id=id_or_slug)
def add_resource(self, resource):
'''Perform an atomic prepend for a new resource'''
self.update(
__raw__={
'$push': {
'resources': {
'$each': [resource.to_mongo()],
'$position': 0
}
}
})
self.reload()
def add_community_resource(self, resource):
'''Perform an atomic prepend for a new resource'''
self.update(
__raw__={
'$push': {
'community_resources': {
'$each': [resource.to_mongo()],
'$position': 0
}
}
})
self.reload()
class OrgRelatedActivity(object):
related_to = db.ReferenceField('Organization')
template = 'activity/organization.html'
class FollowDataset(Follow):
following = db.ReferenceField(Dataset)
class FollowOrg(Follow):
following = db.ReferenceField(Organization)
class User(db.Document, WithMetrics, UserMixin):
slug = db.SlugField(max_length=255,
required=True,
populate_from='fullname')
email = db.StringField(max_length=255, required=True)
password = db.StringField()
active = db.BooleanField()
roles = db.ListField(db.ReferenceField(Role), default=[])
first_name = db.StringField(max_length=255, required=True)
last_name = db.StringField(max_length=255, required=True)
avatar_url = db.URLField()
avatar = db.ImageField(fs=avatars,
basename=default_image_basename,
thumbnails=AVATAR_SIZES)
website = db.URLField()
about = db.StringField()
prefered_language = db.StringField()
apikey = db.StringField()
created_at = db.DateTimeField(default=datetime.now, required=True)
confirmed_at = db.DateTimeField()
last_login_at = db.DateTimeField()
current_login_at = db.DateTimeField()
last_login_ip = db.StringField()
current_login_ip = db.StringField()
login_count = db.IntField()
deleted = db.DateTimeField()
ext = db.MapField(db.GenericEmbeddedDocumentField())
extras = db.ExtrasField()
before_save = Signal()
after_save = Signal()
on_create = Signal()
on_update = Signal()
before_delete = Signal()
after_delete = Signal()
on_delete = Signal()
meta = {
'allow_inheritance': True,
'indexes': ['-created_at', 'slug', 'apikey'],
'ordering': ['-created_at']
}
def __str__(self):
return self.fullname
__unicode__ = __str__
@property
def fullname(self):
return ' '.join((self.first_name or '', self.last_name or '')).strip()
@cached_property
def organizations(self):
from udata.core.organization.models import Organization
return Organization.objects(members__user=self)
@property
def sysadmin(self):
return bool(getattr(g, 'sysadmin'))
@property
def display_url(self):
return url_for('users.show', user=self)
@property
def visible(self):
return self.metrics.get('datasets', 0) + self.metrics.get('reuses',
0) > 0
def generate_api_key(self):
s = JSONWebSignatureSerializer(current_app.config['SECRET_KEY'])
self.apikey = s.dumps({
'user': str(self.id),
'time': time(),
})
def clear_api_key(self):
self.apikey = None
@classmethod
def get(cls, id_or_slug):
obj = cls.objects(slug=id_or_slug).first()
return obj or cls.objects.get_or_404(id=id_or_slug)
@classmethod
def pre_save(cls, sender, document, **kwargs):
cls.before_save.send(document)
@classmethod
def post_save(cls, sender, document, **kwargs):
cls.after_save.send(document)
if kwargs.get('created'):
cls.on_create.send(document)
else:
cls.on_update.send(document)
class Ownable(db.Document):
owner = db.ReferenceField(User)
class Ownable(db.Document):
owner = db.ReferenceField(User)
organization = db.ReferenceField(Organization)
class ReuseIssue(Issue):
subject = db.ReferenceField(Reuse)
class User(WithMetrics, UserMixin, db.Document):
slug = db.SlugField(max_length=255,
required=True,
populate_from='fullname')
email = db.StringField(max_length=255, required=True, unique=True)
password = db.StringField()
active = db.BooleanField()
roles = db.ListField(db.ReferenceField(Role), default=[])
first_name = db.StringField(max_length=255, required=True)
last_name = db.StringField(max_length=255, required=True)
avatar_url = db.URLField()
avatar = db.ImageField(fs=avatars,
basename=default_image_basename,
thumbnails=AVATAR_SIZES)
website = db.URLField()
about = db.StringField()
prefered_language = db.StringField()
apikey = db.StringField()
created_at = db.DateTimeField(default=datetime.now, required=True)
# The field below is required for Flask-security
# when SECURITY_CONFIRMABLE is True
confirmed_at = db.DateTimeField()
password_rotation_demanded = db.DateTimeField()
password_rotation_performed = db.DateTimeField()
# The 5 fields below are required for Flask-security
# when SECURITY_TRACKABLE is True
last_login_at = db.DateTimeField()
current_login_at = db.DateTimeField()
last_login_ip = db.StringField()
current_login_ip = db.StringField()
login_count = db.IntField()
deleted = db.DateTimeField()
ext = db.MapField(db.GenericEmbeddedDocumentField())
extras = db.ExtrasField()
before_save = Signal()
after_save = Signal()
on_create = Signal()
on_update = Signal()
before_delete = Signal()
after_delete = Signal()
on_delete = Signal()
meta = {
'indexes': ['-created_at', 'slug', 'apikey'],
'ordering': ['-created_at']
}
__search_metrics__ = Object(
properties={
'datasets': Integer(),
'reuses': Integer(),
'followers': Integer(),
'views': Integer()
})
__metrics_keys__ = [
'datasets',
'reuses',
'following',
'followers',
]
def __str__(self):
return self.fullname
@property
def fullname(self):
return ' '.join((self.first_name or '', self.last_name or '')).strip()
@cached_property
def organizations(self):
from udata.core.organization.models import Organization
return Organization.objects(members__user=self, deleted__exists=False)
@property
def sysadmin(self):
return self.has_role('admin')
def url_for(self, *args, **kwargs):
return url_for('users.show', user=self, *args, **kwargs)
display_url = property(url_for)
@property
def external_url(self):
return self.url_for(_external=True)
@property
def visible(self):
count = self.metrics.get('datasets', 0) + self.metrics.get('reuses', 0)
return count > 0 and self.active
@cached_property
def resources_availability(self):
"""Return the percentage of availability for resources."""
# Flatten the list.
availabilities = list(
chain(*[org.check_availability() for org in self.organizations]))
# Filter out the unknown
availabilities = [a for a in availabilities if type(a) is bool]
if availabilities:
# Trick will work because it's a sum() of booleans.
return round(100. * sum(availabilities) / len(availabilities), 2)
# if nothing is unavailable, everything is considered OK
return 100
@cached_property
def datasets_org_count(self):
"""Return the number of datasets of user's organizations."""
from udata.models import Dataset # Circular imports.
return sum(
Dataset.objects(organization=org).visible().count()
for org in self.organizations)
@cached_property
def followers_org_count(self):
"""Return the number of followers of user's organizations."""
from udata.models import Follow # Circular imports.
return sum(
Follow.objects(following=org).count()
for org in self.organizations)
@property
def datasets_count(self):
"""Return the number of datasets of the user."""
return self.metrics.get('datasets', 0)
@property
def followers_count(self):
"""Return the number of followers of the user."""
return self.metrics.get('followers', 0)
def generate_api_key(self):
s = JSONWebSignatureSerializer(current_app.config['SECRET_KEY'])
byte_str = s.dumps({
'user': str(self.id),
'time': time(),
})
self.apikey = byte_str.decode()
def clear_api_key(self):
self.apikey = None
@classmethod
def get(cls, id_or_slug):
obj = cls.objects(slug=id_or_slug).first()
return obj or cls.objects.get_or_404(id=id_or_slug)
@classmethod
def pre_save(cls, sender, document, **kwargs):
cls.before_save.send(document)
@classmethod
def post_save(cls, sender, document, **kwargs):
cls.after_save.send(document)
if kwargs.get('created'):
cls.on_create.send(document)
else:
cls.on_update.send(document)
@cached_property
def json_ld(self):
result = {
'@type': 'Person',
'@context': 'http://schema.org',
'name': self.fullname,
}
if self.about:
result['description'] = mdstrip(self.about)
if self.avatar_url:
result['image'] = self.avatar_url
if self.website:
result['url'] = self.website
return result
def mark_as_deleted(self):
copied_user = copy(self)
self.email = '{}@deleted'.format(self.id)
self.slug = 'deleted'
self.password = None
self.active = False
self.first_name = 'DELETED'
self.last_name = 'DELETED'
self.avatar = None
self.avatar_url = None
self.website = None
self.about = None
self.extras = None
self.apikey = None
self.deleted = datetime.now()
self.save()
for organization in self.organizations:
organization.members = [
member for member in organization.members
if member.user != self
]
organization.save()
for discussion in Discussion.objects(discussion__posted_by=self):
for message in discussion.discussion:
if message.posted_by == self:
message.content = 'DELETED'
discussion.save()
Follow.objects(follower=self).delete()
Follow.objects(following=self).delete()
mail.send(_('Account deletion'), copied_user, 'account_deleted')
def count_datasets(self):
from udata.models import Dataset
self.metrics['datasets'] = Dataset.objects(
owner=self).visible().count()
self.save()
def count_reuses(self):
from udata.models import Reuse
self.metrics['reuses'] = Reuse.objects(owner=self).visible().count()
self.save()
def count_followers(self):
from udata.models import Follow
self.metrics['followers'] = Follow.objects(
until=None).followers(self).count()
self.save()
def count_following(self):
from udata.models import Follow
self.metrics['following'] = Follow.objects.following(self).count()
self.save()
class FollowReuse(Follow):
following = db.ReferenceField(Reuse)
class FollowFake(Follow):
following = db.ReferenceField(Fake)
class Team(db.EmbeddedDocument):
name = db.StringField(required=True)
slug = db.SlugField(max_length=255, required=True, populate_from='name', update=True, unique=False)
description = db.StringField()
members = db.ListField(db.ReferenceField('User'))
