class Resource(WithMetrics, db.EmbeddedDocument):
id = db.AutoUUIDField()
title = db.StringField(verbose_name="Title", required=True)
description = db.StringField()
type = db.StringField(choices=RESOURCE_TYPES.keys(),
default='file',
required=True)
url = db.StringField()
urlhash = db.StringField()
checksum = db.EmbeddedDocumentField(Checksum)
format = db.StringField()
mime = db.StringField()
size = db.IntField()
owner = db.ReferenceField('User')
created_at = db.DateTimeField(default=datetime.datetime.now, required=True)
modified = db.DateTimeField(default=datetime.datetime.now, required=True)
published = db.DateTimeField(default=datetime.datetime.now, required=True)
deleted = db.DateTimeField()
on_added = Signal()
on_deleted = Signal()
def clean(self):
super(Resource, self).clean()
if not self.urlhash or 'url' in self._get_changed_fields():
self.urlhash = hash_url(self.url)
class ResourceMixin(object):
id = db.AutoUUIDField(primary_key=True)
title = db.StringField(verbose_name="Title", required=True)
description = db.StringField()
filetype = db.StringField(choices=RESOURCE_TYPES.keys(),
default='file',
required=True)
url = db.StringField()
urlhash = db.StringField()
checksum = db.EmbeddedDocumentField(Checksum)
format = db.StringField()
mime = db.StringField()
filesize = db.IntField() # `size` is a reserved keyword for mongoengine.
created_at = db.DateTimeField(default=datetime.now, required=True)
modified = db.DateTimeField(default=datetime.now, required=True)
published = db.DateTimeField(default=datetime.now, required=True)
deleted = db.DateTimeField()
def clean(self):
super(ResourceMixin, self).clean()
if not self.urlhash or 'url' in self._get_changed_fields():
self.urlhash = hash_url(self.url)
@property
def closed_format(self):
"""Return True if the specified format is in CLOSED_FORMATS."""
return self.format.lower() in CLOSED_FORMATS
def check_availability(self, group):
"""Check if a resource is reachable against a Croquemort server.
Return a boolean.
"""
if self.filetype == 'remote':
# We perform a quick check for performances matters.
error, response = check_url_from_cache(self.url, group)
if error or int(response.get('status', 500)) >= 500:
return False
else:
return True
else:
return True # We consider that API cases (types) are OK.
@property
def is_available(self):
return self.check_availability(group=None)
class MembershipRequest(db.EmbeddedDocument):
'''
Pending organization membership requests
'''
id = db.AutoUUIDField()
user = db.ReferenceField('User')
status = db.StringField(choices=list(MEMBERSHIP_STATUS), default='pending')
created = db.DateTimeField(default=datetime.now, required=True)
handled_on = db.DateTimeField()
handled_by = db.ReferenceField('User')
comment = db.StringField()
refusal_comment = db.StringField()
@property
def status_label(self):
return MEMBERSHIP_STATUS[self.status]
class ResourceMixin(object):
id = db.AutoUUIDField(primary_key=True)
title = db.StringField(verbose_name="Title", required=True)
description = db.StringField()
filetype = db.StringField(
choices=RESOURCE_FILETYPES.keys(), default='file', required=True)
type = db.StringField(
choices=RESOURCE_TYPES.keys(), default='main', required=True)
url = db.URLField(required=True)
urlhash = db.StringField()
checksum = db.EmbeddedDocumentField(Checksum)
format = db.StringField()
mime = db.StringField()
filesize = db.IntField() # `size` is a reserved keyword for mongoengine.
extras = db.ExtrasField()
created_at = db.DateTimeField(default=datetime.now, required=True)
modified = db.DateTimeField(default=datetime.now, required=True)
published = db.DateTimeField(default=datetime.now, required=True)
deleted = db.DateTimeField()
def clean(self):
super(ResourceMixin, self).clean()
if not self.urlhash or 'url' in self._get_changed_fields():
self.urlhash = hash_url(self.url)
@cached_property # Accessed at least 2 times in front rendering
def preview_url(self):
return get_preview_url(self)
@property
def closed_or_no_format(self):
"""
Return True if the specified format is in CLOSED_FORMATS or
no format has been specified.
"""
return not self.format or self.format.lower() in CLOSED_FORMATS
def check_availability(self):
'''
Return the check status from extras if any.
NB: `unknown` will evaluate to True in the aggregate checks using
`all([])` (dataset, organization, user).
'''
return self.extras.get('check:available', 'unknown')
def need_check(self):
'''Does the resource needs to be checked against its linkchecker?
We check unavailable resources often, unless they go over the
threshold. Available resources are checked less and less frequently
based on their historical availability.
'''
min_cache_duration, max_cache_duration, ko_threshold = [
current_app.config.get(k) for k in (
'LINKCHECKING_MIN_CACHE_DURATION',
'LINKCHECKING_MAX_CACHE_DURATION',
'LINKCHECKING_UNAVAILABLE_THRESHOLD',
)
]
count_availability = self.extras.get('check:count-availability', 1)
is_available = self.check_availability()
if is_available == 'unknown':
return True
elif is_available or count_availability > ko_threshold:
delta = min(min_cache_duration * count_availability,
max_cache_duration)
else:
delta = min_cache_duration
if self.extras.get('check:date'):
limit_date = datetime.now() - timedelta(minutes=delta)
check_date = self.extras['check:date']
if not isinstance(check_date, datetime):
try:
check_date = parse_dt(check_date)
except (ValueError, TypeError):
return True
if check_date >= limit_date:
return False
return True
@property
def latest(self):
'''
Permanent link to the latest version of this resource.
If this resource is updated and `url` changes, this property won't.
'''
return url_for('datasets.resource', id=self.id, _external=True)
@cached_property
def json_ld(self):
result = {
'@type': 'DataDownload',
'@id': str(self.id),
'url': self.latest,
'name': self.title or _('Nameless resource'),
'contentUrl': self.url,
'dateCreated': self.created_at.isoformat(),
'dateModified': self.modified.isoformat(),
'datePublished': self.published.isoformat(),
'extras': [get_json_ld_extra(*item)
for item in self.extras.items()],
}
if 'views' in self.metrics:
result['interactionStatistic'] = {
'@type': 'InteractionCounter',
'interactionType': {
'@type': 'DownloadAction',
},
'userInteractionCount': self.metrics['views']
}
if self.format:
result['encodingFormat'] = self.format
if self.filesize:
result['contentSize'] = self.filesize
if self.mime:
result['fileFormat'] = self.mime
if self.description:
result['description'] = mdstrip(self.description)
return result
class UUIDAsIdTester(db.Document):
id = db.AutoUUIDField(primary_key=True)
class UUIDTester(db.Document):
uuid = db.AutoUUIDField()
class Fake(db.Document):
id = db.AutoUUIDField()
class Nested(db.EmbeddedDocument):
id = db.AutoUUIDField()
name = db.StringField()
sub = db.EmbeddedDocumentField(SubNested)
class Nested(db.EmbeddedDocument):
id = db.AutoUUIDField()
name = db.StringField(required=True)
class ResourceMixin(object):
id = db.AutoUUIDField(primary_key=True)
title = db.StringField(verbose_name="Title", required=True)
description = db.StringField()
filetype = db.StringField(choices=RESOURCE_TYPES.keys(),
default='file',
required=True)
url = db.URLField(required=True)
urlhash = db.StringField()
checksum = db.EmbeddedDocumentField(Checksum)
format = db.StringField()
mime = db.StringField()
filesize = db.IntField() # `size` is a reserved keyword for mongoengine.
extras = db.ExtrasField()
created_at = db.DateTimeField(default=datetime.now, required=True)
modified = db.DateTimeField(default=datetime.now, required=True)
published = db.DateTimeField(default=datetime.now, required=True)
deleted = db.DateTimeField()
def clean(self):
super(ResourceMixin, self).clean()
if not self.urlhash or 'url' in self._get_changed_fields():
self.urlhash = hash_url(self.url)
@property
def closed_format(self):
"""Return True if the specified format is in CLOSED_FORMATS."""
return self.format and self.format.lower() in CLOSED_FORMATS
def check_availability(self):
'''
Return the check status from extras if any.
NB: `unknown` will evaluate to True in the aggregate checks using
`all([])` (dataset, organization, user).
'''
return self.extras.get('check:available', 'unknown')
@property
def latest(self):
'''
Permanent link to the latest version of this resource.
If this resource is updated and `url` changes, this property won't.
'''
return url_for('datasets.resource', id=self.id, _external=True)
@cached_property
def json_ld(self):
result = {
'@type': 'DataDownload',
'@id': str(self.id),
'url': self.latest,
'name': self.title or _('Nameless resource'),
'contentUrl': self.url,
'dateCreated': self.created_at.isoformat(),
'dateModified': self.modified.isoformat(),
'datePublished': self.published.isoformat(),
'extras':
[get_json_ld_extra(*item) for item in self.extras.items()],
}
if 'views' in self.metrics:
result['interactionStatistic'] = {
'@type': 'InteractionCounter',
'interactionType': {
'@type': 'DownloadAction',
},
'userInteractionCount': self.metrics['views']
}
if self.format:
result['encodingFormat'] = self.format
if self.filesize:
result['contentSize'] = self.filesize
if self.mime:
result['fileFormat'] = self.mime
if self.description:
result['description'] = mdstrip(self.description)
# These 2 values are not standard
if self.checksum:
result['checksum'] = self.checksum.value,
result['checksumType'] = self.checksum.type or 'sha1'
return result
class ResourceMixin(object):
id = db.AutoUUIDField(primary_key=True)
title = db.StringField(verbose_name="Title", required=True)
description = db.StringField()
filetype = db.StringField(choices=RESOURCE_TYPES.keys(),
default='file',
required=True)
url = db.StringField()
urlhash = db.StringField()
checksum = db.EmbeddedDocumentField(Checksum)
format = db.StringField()
mime = db.StringField()
filesize = db.IntField() # `size` is a reserved keyword for mongoengine.
created_at = db.DateTimeField(default=datetime.now, required=True)
modified = db.DateTimeField(default=datetime.now, required=True)
published = db.DateTimeField(default=datetime.now, required=True)
deleted = db.DateTimeField()
def clean(self):
super(ResourceMixin, self).clean()
if not self.urlhash or 'url' in self._get_changed_fields():
self.urlhash = hash_url(self.url)
@property
def closed_format(self):
"""Return True if the specified format is in CLOSED_FORMATS."""
return self.format.lower() in CLOSED_FORMATS
def check_availability(self, group):
"""Check if a resource is reachable against a Croquemort server.
Return a boolean.
"""
if self.filetype == 'remote':
# We perform a quick check for performances matters.
error, response = check_url_from_cache(self.url, group)
if error or 'status' not in response:
return False
elif int(response['status']) >= httplib.INTERNAL_SERVER_ERROR:
return False
else:
return True
else:
return True # We consider that API cases (types) are OK.
@property
def is_available(self):
return self.check_availability(group=None)
@property
def latest(self):
'''
Permanent link to the latest version of this resource.
If this resource is updated and `url` changes, this property won't.
'''
return url_for('datasets.resource', id=self.id, _external=True)
@cached_property
def json_ld(self):
result = {
'@type': 'DataDownload',
'@id': str(self.id),
'url': self.latest,
'name': self.title or _('Nameless resource'),
'contentUrl': self.url,
'dateCreated': self.created_at.isoformat(),
'dateModified': self.modified.isoformat(),
'datePublished': self.published.isoformat(),
}
if 'views' in self.metrics:
result['interactionStatistic'] = {
'@type': 'InteractionCounter',
'interactionType': {
'@type': 'DownloadAction',
},
'userInteractionCount': self.metrics['views']
}
if self.format:
result['encodingFormat'] = self.format
if self.filesize:
result['contentSize'] = self.filesize
if self.mime:
result['fileFormat'] = self.mime
if self.description:
result['description'] = mdstrip(self.description)
# These 2 values are not standard
if self.checksum:
result['checksum'] = self.checksum.value,
result['checksumType'] = self.checksum.type or 'sha1'
return result
