class Discussion(db.Document):
user = db.ReferenceField('User')
subject = db.GenericReferenceField()
title = db.StringField(required=True)
discussion = db.ListField(db.EmbeddedDocumentField(Message))
created = db.DateTimeField(default=datetime.now, required=True)
closed = db.DateTimeField()
closed_by = db.ReferenceField('User')
extras = db.ExtrasField()
meta = {
'indexes': [
'user',
'subject',
'-created'
],
'ordering': ['-created'],
}
def person_involved(self, person):
"""Return True if the given person has been involved in the
discussion, False otherwise.
"""
return any(message.posted_by == person for message in self.discussion)
@property
def external_url(self):
return self.subject.url_for(
_anchor='discussion-{id}'.format(id=self.id),
_external=True)
class Resource(WithMetrics, db.EmbeddedDocument):
id = db.AutoUUIDField()
title = db.StringField(verbose_name="Title", required=True)
description = db.StringField()
type = db.StringField(choices=RESOURCE_TYPES.keys(),
default='file',
required=True)
url = db.StringField()
urlhash = db.StringField()
checksum = db.EmbeddedDocumentField(Checksum)
format = db.StringField()
mime = db.StringField()
size = db.IntField()
owner = db.ReferenceField('User')
created_at = db.DateTimeField(default=datetime.datetime.now, required=True)
modified = db.DateTimeField(default=datetime.datetime.now, required=True)
published = db.DateTimeField(default=datetime.datetime.now, required=True)
deleted = db.DateTimeField()
on_added = Signal()
on_deleted = Signal()
def clean(self):
super(Resource, self).clean()
if not self.urlhash or 'url' in self._get_changed_fields():
self.urlhash = hash_url(self.url)
class HarvestSource(db.Owned, db.Document):
name = db.StringField(max_length=255)
slug = db.SlugField(max_length=255,
required=True,
unique=True,
populate_from='name',
update=True)
description = db.StringField()
url = db.StringField(required=True)
backend = db.StringField()
config = db.DictField()
periodic_task = db.ReferenceField('PeriodicTask',
reverse_delete_rule=db.NULLIFY)
created_at = db.DateTimeField(default=datetime.now, required=True)
frequency = db.StringField(choices=HARVEST_FREQUENCIES.keys(),
default=DEFAULT_HARVEST_FREQUENCY,
required=True)
active = db.BooleanField(default=True)
validation = db.EmbeddedDocumentField(HarvestSourceValidation,
default=HarvestSourceValidation)
deleted = db.DateTimeField()
@property
def domain(self):
parsed = urlparse(self.url)
return parsed.netloc.split(':')[0]
@classmethod
def get(cls, ident):
return cls.objects(slug=ident).first() or cls.objects.get(pk=ident)
def get_last_job(self):
return HarvestJob.objects(source=self).order_by('-created').first()
@cached_property
def last_job(self):
return self.get_last_job()
@property
def schedule(self):
if not self.periodic_task:
return
return self.periodic_task.schedule_display
meta = {
'indexes': [
'-created_at',
'slug',
('deleted', '-created_at'),
] + db.Owned.meta['indexes'],
'ordering': ['-created_at'],
'queryset_class':
HarvestSourceQuerySet,
}
def __unicode__(self):
return self.name or ''
class HarvestJob(db.Document):
'''Keep track of harvestings'''
created = db.DateTimeField(default=datetime.now, required=True)
started = db.DateTimeField()
ended = db.DateTimeField()
status = db.StringField(choices=HARVEST_JOB_STATUS.keys(),
default=DEFAULT_HARVEST_JOB_STATUS, required=True)
errors = db.ListField(db.EmbeddedDocumentField(HarvestError))
items = db.ListField(db.EmbeddedDocumentField(HarvestItem))
source = db.ReferenceField(HarvestSource, reverse_delete_rule=db.NULLIFY)
class HarvestItem(db.EmbeddedDocument):
remote_id = db.StringField()
dataset = db.ReferenceField(Dataset)
status = db.StringField(choices=HARVEST_ITEM_STATUS.keys(),
default=DEFAULT_HARVEST_ITEM_STATUS, required=True)
created = db.DateTimeField(default=datetime.now, required=True)
started = db.DateTimeField()
ended = db.DateTimeField()
errors = db.ListField(db.EmbeddedDocumentField(HarvestError))
args = db.ListField(db.StringField())
kwargs = db.DictField()
class Issue(db.Document):
user = db.ReferenceField('User')
subject = db.GenericReferenceField()
title = db.StringField(required=True)
discussion = db.ListField(db.EmbeddedDocumentField(Message))
created = db.DateTimeField(default=datetime.now, required=True)
closed = db.DateTimeField()
closed_by = db.ReferenceField('User')
meta = {
'indexes': ['user', 'subject', '-created'],
'ordering': ['-created'],
}
class Discussion(db.Document):
user = db.ReferenceField('User')
subject = db.ReferenceField(db.DomainModel)
title = db.StringField(required=True)
discussion = db.ListField(db.EmbeddedDocumentField(Message))
created = db.DateTimeField(default=datetime.now, required=True)
closed = db.DateTimeField()
closed_by = db.ReferenceField('User')
meta = {
'indexes': ['user', 'subject', 'created'],
'allow_inheritance': True,
'ordering': ['created'],
}
class OAuth2Grant(db.Document):
user = db.ReferenceField('User', required=True)
client = db.ReferenceField('OAuth2Client', required=True)
code = db.StringField(required=True)
redirect_uri = db.StringField()
expires = db.DateTimeField()
scopes = db.ListField(db.StringField())
meta = {
'collection': 'oauth2_grant'
}
def __str__(self):
return '<OAuth2Grant({0.client.name}, {0.user.fullname})>'.format(self)
def is_expired(self):
return self.expires < datetime.utcnow()
def get_redirect_uri(self):
return self.redirect_uri
def get_scope(self):
return ' '.join(self.scopes)
class Badge(db.EmbeddedDocument):
kind = db.StringField(choices=[], required=True)
created = db.DateTimeField(default=datetime.now, required=True)
created_by = db.ReferenceField('User')
removed = db.DateTimeField()
removed_by = db.ReferenceField('User')
meta = {
'allow_inheritance': True,
'ordering': ['created'],
}
def __unicode__(self):
return self.kind
__str__ = __unicode__
class OAuth2Code(db.Document):
user = db.ReferenceField('User', required=True)
client = db.ReferenceField('OAuth2Client', required=True)
code = db.StringField(required=True)
redirect_uri = db.StringField()
expires = db.DateTimeField()
scope = db.StringField(default='')
code_challenge = db.StringField()
code_challenge_method = db.StringField()
meta = {'collection': 'oauth2_code'}
def __str__(self):
return '<OAuth2Code({0.client.name}, {0.user.fullname})>'.format(self)
def is_expired(self):
return self.expires < datetime.utcnow()
def get_redirect_uri(self):
return self.redirect_uri
def get_scope(self):
return self.scope
class HarvestSource(db.Document):
name = db.StringField(max_length=255)
slug = db.SlugField(max_length=255,
required=True,
unique=True,
populate_from='name',
update=True)
description = db.StringField()
url = db.StringField()
backend = db.StringField()
config = db.DictField()
periodic_task = db.ReferenceField('PeriodicTask',
reverse_delete_rule=db.NULLIFY)
created_at = db.DateTimeField(default=datetime.now, required=True)
frequency = db.StringField(choices=HARVEST_FREQUENCIES.keys(),
default=DEFAULT_HARVEST_FREQUENCY,
required=True)
active = db.BooleanField(default=True)
owner = db.ReferenceField('User', reverse_delete_rule=db.NULLIFY)
organization = db.ReferenceField('Organization',
reverse_delete_rule=db.NULLIFY)
@classmethod
def get(cls, ident):
return cls.objects(slug=ident).first() or cls.objects.get(id=ident)
def get_last_job(self):
return HarvestJob.objects(source=self).order_by('-created')[0]
class OAuth2Token(db.Document):
client = db.ReferenceField('OAuth2Client', required=True)
user = db.ReferenceField('User')
# currently only bearer is supported
token_type = db.StringField(choices=TOKEN_TYPES.keys(), default='Bearer')
access_token = db.StringField(unique=True)
refresh_token = db.StringField(unique=True, sparse=True)
created_at = db.DateTimeField(default=datetime.utcnow, required=True)
expires_in = db.IntField(required=True, default=TOKEN_EXPIRATION)
scopes = db.ListField(db.StringField())
meta = {'collection': 'oauth2_token'}
def __unicode__(self):
return '<OAuth2Token({0.client.name})>'.format(self)
def get_scope(self):
return ' '.join(self.scopes)
def get_expires_in(self):
return self.expires_in
def get_expires_at(self):
return (self.created_at - EPOCH).total_seconds() + self.expires_in
def is_refresh_token_expired(self):
expired_at = datetime.fromtimestamp(self.get_expires_at())
expired_at += timedelta(days=REFRESH_EXPIRATION)
return expired_at < datetime.utcnow()
class Follow(db.Document):
follower = db.ReferenceField('User', required=True)
following = db.GenericReferenceField()
since = db.DateTimeField(required=True, default=datetime.now)
until = db.DateTimeField()
meta = {
'indexes': [
'follower',
'following',
('follower', 'until'),
('following', 'until'),
],
'queryset_class':
FollowQuerySet,
}
class ResourceMixin(object):
id = db.AutoUUIDField(primary_key=True)
title = db.StringField(verbose_name="Title", required=True)
description = db.StringField()
filetype = db.StringField(choices=RESOURCE_TYPES.keys(),
default='file',
required=True)
url = db.StringField()
urlhash = db.StringField()
checksum = db.EmbeddedDocumentField(Checksum)
format = db.StringField()
mime = db.StringField()
filesize = db.IntField() # `size` is a reserved keyword for mongoengine.
created_at = db.DateTimeField(default=datetime.now, required=True)
modified = db.DateTimeField(default=datetime.now, required=True)
published = db.DateTimeField(default=datetime.now, required=True)
deleted = db.DateTimeField()
def clean(self):
super(ResourceMixin, self).clean()
if not self.urlhash or 'url' in self._get_changed_fields():
self.urlhash = hash_url(self.url)
@property
def closed_format(self):
"""Return True if the specified format is in CLOSED_FORMATS."""
return self.format.lower() in CLOSED_FORMATS
def check_availability(self, group):
"""Check if a resource is reachable against a Croquemort server.
Return a boolean.
"""
if self.filetype == 'remote':
# We perform a quick check for performances matters.
error, response = check_url_from_cache(self.url, group)
if error or int(response.get('status', 500)) >= 500:
return False
else:
return True
else:
return True # We consider that API cases (types) are OK.
@property
def is_available(self):
return self.check_availability(group=None)
class HarvestJob(db.Document):
'''Keep track of harvestings'''
created = db.DateTimeField(default=datetime.now, required=True)
started = db.DateTimeField()
ended = db.DateTimeField()
status = db.StringField(choices=HARVEST_JOB_STATUS.keys(),
default=DEFAULT_HARVEST_JOB_STATUS,
required=True)
errors = db.ListField(db.EmbeddedDocumentField(HarvestError))
items = db.ListField(db.EmbeddedDocumentField(HarvestItem))
source = db.ReferenceField(HarvestSource, reverse_delete_rule=db.CASCADE)
data = db.DictField()
meta = {
'indexes': ['-created', 'source', ('source', '-created')],
'ordering': ['-created'],
}
class Member(db.EmbeddedDocument):
user = db.ReferenceField('User')
role = db.StringField(choices=list(ORG_ROLES), default=DEFAULT_ROLE)
since = db.DateTimeField(default=datetime.now, required=True)
@property
def label(self):
return ORG_ROLES[self.role]
class HarvestSourceValidation(db.EmbeddedDocument):
'''Store harvest source validation details'''
state = db.StringField(choices=VALIDATION_STATES.keys(),
default=VALIDATION_PENDING,
required=True)
by = db.ReferenceField('User')
on = db.DateTimeField()
comment = db.StringField()
class License(db.Document):
# We need to declare id explicitly since we do not use the default
# value set by Mongo.
id = db.StringField(primary_key=True)
created_at = db.DateTimeField(default=datetime.now, required=True)
title = db.StringField(required=True)
slug = db.SlugField(required=True, populate_from='title')
url = db.URLField()
maintainer = db.StringField()
flags = db.ListField(db.StringField())
active = db.BooleanField()
def __unicode__(self):
return self.title
@classmethod
def guess(cls, *strings, **kwargs):
'''
Try to guess a license from a list of strings.
Accept a `default` keyword argument which will be
the default fallback license.
'''
license = None
for string in strings:
license = cls.guess_one(string)
if license:
break
return license or kwargs.get('default')
@classmethod
def guess_one(cls, text):
'''
Try to guess license from a string.
Try to exact match on identifier then slugified title
and fallback on edit distance ranking (after slugification)
'''
if not text:
return
qs = cls.objects
text = text.strip().lower() # Stored identifiers are lower case
slug = cls.slug.slugify(text) # Use slug as it normalize string
license = qs(db.Q(id=text) | db.Q(slug=slug) | db.Q(url=text)).first()
if license is None:
# Try to single match with a low Damerau-Levenshtein distance
computed = ((l, rdlevenshtein(l.slug, slug)) for l in cls.objects)
candidates = [l for l, d in computed if d <= MAX_DISTANCE]
# If there is more that one match, we cannot determinate
# which one is closer to safely choose between candidates
if len(candidates) == 1:
license = candidates[0]
return license
@classmethod
def default(cls):
return cls.objects(id=DEFAULT_LICENSE['id']).first()
class PiwikTracking(db.Document):
url = db.StringField(required=True)
date = db.DateTimeField(required=True, default=datetime.now)
kwargs = db.DictField()
meta = {
'indexes': ['date'],
'ordering': ['date'],
}
class MembershipRequest(db.EmbeddedDocument):
'''
Pending organization membership requests
'''
id = db.AutoUUIDField()
user = db.ReferenceField('User')
status = db.StringField(choices=list(MEMBERSHIP_STATUS), default='pending')
created = db.DateTimeField(default=datetime.now, required=True)
handled_on = db.DateTimeField()
handled_by = db.ReferenceField('User')
comment = db.StringField()
refusal_comment = db.StringField()
@property
def status_label(self):
return MEMBERSHIP_STATUS[self.status]
class Post(db.Datetimed, db.Document):
name = db.StringField(max_length=255, required=True)
slug = db.SlugField(max_length=255,
required=True,
populate_from='name',
update=True,
follow=True)
headline = db.StringField()
content = db.StringField(required=True)
image_url = db.StringField()
image = db.ImageField(fs=images,
basename=default_image_basename,
thumbnails=IMAGE_SIZES)
credit_to = db.StringField()
credit_url = db.URLField()
tags = db.ListField(db.StringField())
datasets = db.ListField(
db.ReferenceField('Dataset', reverse_delete_rule=db.PULL))
reuses = db.ListField(
db.ReferenceField('Reuse', reverse_delete_rule=db.PULL))
owner = db.ReferenceField('User')
published = db.DateTimeField()
meta = {
'ordering': ['-created_at'],
'indexes': [
'-created_at',
'-published',
],
'queryset_class': PostQuerySet,
}
verbose_name = _('post')
def __str__(self):
return self.name or ''
def url_for(self, *args, **kwargs):
return url_for('posts.show', post=self, *args, **kwargs)
@property
def display_url(self):
return self.url_for()
@property
def external_url(self):
return self.url_for(_external=True)
def count_discussions(self):
# There are no metrics on Post to store discussions count
pass
class Discussion(db.Document):
user = db.ReferenceField('User')
subject = db.ReferenceField(db.DomainModel)
title = db.StringField(required=True)
discussion = db.ListField(db.EmbeddedDocumentField(Message))
created = db.DateTimeField(default=datetime.now, required=True)
closed = db.DateTimeField()
closed_by = db.ReferenceField('User')
meta = {
'indexes': ['user', 'subject', 'created'],
'allow_inheritance': True,
'ordering': ['created'],
}
def person_involved(self, person):
"""Return True if the given person has been involved in the
discussion, False otherwise.
"""
return any(message.posted_by == person for message in self.discussion)
class OAuth2Grant(db.Document):
user = db.ReferenceField('User', required=True)
client = db.ReferenceField('OAuth2Client', required=True)
code = db.StringField(required=True)
redirect_uri = db.StringField()
expires = db.DateTimeField()
scopes = db.ListField(db.StringField())
meta = {'collection': 'oauth2_grant'}
class Transfer(db.Document):
owner = db.GenericReferenceField(required=True)
recipient = db.GenericReferenceField(required=True)
subject = db.GenericReferenceField(required=True)
comment = db.StringField()
status = db.StringField(choices=list(TRANSFER_STATUS), default='pending')
created = db.DateTimeField(default=datetime.now, required=True)
responded = db.DateTimeField()
responder = db.ReferenceField('User')
response_comment = db.StringField()
meta = {
'indexes': [
'owner',
'recipient',
'created',
'status',
]
}
class License(db.Document):
id = db.StringField(primary_key=True)
created_at = db.DateTimeField(default=datetime.datetime.now, required=True)
title = db.StringField(required=True)
slug = db.SlugField(required=True, populate_from='title')
url = db.URLField()
maintainer = db.StringField()
flags = db.ListField(db.StringField())
active = db.BooleanField()
def __unicode__(self):
return self.title
class Badge(db.EmbeddedDocument):
kind = db.StringField(required=True)
created = db.DateTimeField(default=datetime.now, required=True)
created_by = db.ReferenceField('User')
def __str__(self):
return self.kind
def validate(self, clean=True):
badges = getattr(self._instance, '__badges__', {})
if self.kind not in badges.keys():
raise db.ValidationError('Unknown badge type %s' % self.kind)
return super(Badge, self).validate(clean=clean)
class Issue(db.Document):
user = db.ReferenceField('User')
subject = db.ReferenceField(db.DomainModel)
type = db.StringField(choices=ISSUE_TYPES.keys())
discussion = db.ListField(db.EmbeddedDocumentField(Message))
created = db.DateTimeField(default=datetime.now, required=True)
closed = db.DateTimeField()
closed_by = db.ReferenceField('User')
meta = {
'indexes': ['user', 'subject', 'created'],
'allow_inheritance': True,
}
@property
def type_label(self):
return ISSUE_TYPES[self.type]
@property
def description(self):
return self.discussion[0].content
class OAuth2Token(db.Document):
client = db.ReferenceField('OAuth2Client', required=True)
user = db.ReferenceField('User')
# currently only bearer is supported
token_type = db.StringField(choices=TOKEN_TYPES.keys(), default='Bearer')
access_token = db.StringField(unique=True)
refresh_token = db.StringField(unique=True)
expires = db.DateTimeField(
default=lambda: datetime.utcnow() + timedelta(hours=TOKEN_EXPIRATION))
scopes = db.ListField(db.StringField())
meta = {'collection': 'oauth2_token'}
class License(db.Document):
# We need to declare id explicitly since we do not use the default
# value set by Mongo.
id = db.StringField(primary_key=True)
created_at = db.DateTimeField(default=datetime.now, required=True)
title = db.StringField(required=True)
slug = db.SlugField(required=True, populate_from='title')
url = db.URLField()
maintainer = db.StringField()
flags = db.ListField(db.StringField())
active = db.BooleanField()
def __unicode__(self):
return self.title
class Activity(db.Document):
'''Store the activity entries for a single related object'''
actor = db.ReferenceField('User', required=True)
organization = db.ReferenceField('Organization')
related_to = db.ReferenceField(db.DomainModel, required=True)
created_at = db.DateTimeField(default=datetime.now, required=True)
kwargs = db.DictField()
on_new = Signal()
__metaclass__ = EmitNewActivityMetaClass
meta = {
'indexes': [
'actor',
'organization',
'related_to',
'-created_at',
('actor', '-created_at'),
('organization', '-created_at'),
('related_to', '-created_at'),
],
'allow_inheritance': True,
}
key = None
label = None
badge_type = 'primary'
icon = 'fa fa-info-circle'
template = 'activity/base.html'
@classmethod
def connect(cls, func):
return cls.on_new.connect(func, sender=cls)
@classmethod
def emit(cls, related_to, organization=None, **kwargs):
return cls.objects.create(
actor=current_user._get_current_object(),
related_to=related_to,
organization=organization
)
