class Site(WithMetrics, db.Document):
id = db.StringField(primary_key=True)
title = db.StringField(required=True)
keywords = db.ListField(db.StringField())
feed_size = db.IntField(required=True, default=DEFAULT_FEED_SIZE)
configs = db.DictField()
themes = db.DictField()
settings = db.EmbeddedDocumentField(SiteSettings, default=SiteSettings)
class Metrics(db.Document):
object_id = db.DynamicField(required=True, null=False, unique_with='date')
date = db.StringField(required=True)
level = db.StringField(
required=True, default='daily',
choices=('hourly', 'daily', 'monthly', 'yearly'))
values = db.DictField()
meta = {
'indexes': [
'object_id',
'level',
'date',
{
'fields': ('object_id', 'level', '-date'),
'unique': True,
}
],
'queryset_class': MetricsQuerySet,
}
def __unicode__(self):
return 'Metrics for object {0} on {1} ({2})'.format(
self.object_id, self.date, self.level
)
class WithMetrics(object):
metrics = db.DictField()
__metrics_keys__ = []
def get_metrics(self):
return {key: self.metrics.get(key, 0) for key in self.__metrics_keys__}
class HarvestSource(db.Document):
name = db.StringField(max_length=255)
slug = db.SlugField(max_length=255,
required=True,
unique=True,
populate_from='name',
update=True)
description = db.StringField()
url = db.StringField()
backend = db.StringField()
config = db.DictField()
periodic_task = db.ReferenceField('PeriodicTask',
reverse_delete_rule=db.NULLIFY)
created_at = db.DateTimeField(default=datetime.now, required=True)
frequency = db.StringField(choices=HARVEST_FREQUENCIES.keys(),
default=DEFAULT_HARVEST_FREQUENCY,
required=True)
active = db.BooleanField(default=True)
owner = db.ReferenceField('User', reverse_delete_rule=db.NULLIFY)
organization = db.ReferenceField('Organization',
reverse_delete_rule=db.NULLIFY)
@classmethod
def get(cls, ident):
return cls.objects(slug=ident).first() or cls.objects.get(id=ident)
def get_last_job(self):
return HarvestJob.objects(source=self).order_by('-created')[0]
class HarvestSource(db.Owned, db.Document):
name = db.StringField(max_length=255)
slug = db.SlugField(max_length=255,
required=True,
unique=True,
populate_from='name',
update=True)
description = db.StringField()
url = db.StringField(required=True)
backend = db.StringField()
config = db.DictField()
periodic_task = db.ReferenceField('PeriodicTask',
reverse_delete_rule=db.NULLIFY)
created_at = db.DateTimeField(default=datetime.now, required=True)
frequency = db.StringField(choices=HARVEST_FREQUENCIES.keys(),
default=DEFAULT_HARVEST_FREQUENCY,
required=True)
active = db.BooleanField(default=True)
validation = db.EmbeddedDocumentField(HarvestSourceValidation,
default=HarvestSourceValidation)
deleted = db.DateTimeField()
@property
def domain(self):
parsed = urlparse(self.url)
return parsed.netloc.split(':')[0]
@classmethod
def get(cls, ident):
return cls.objects(slug=ident).first() or cls.objects.get(pk=ident)
def get_last_job(self):
return HarvestJob.objects(source=self).order_by('-created').first()
@cached_property
def last_job(self):
return self.get_last_job()
@property
def schedule(self):
if not self.periodic_task:
return
return self.periodic_task.schedule_display
meta = {
'indexes': [
'-created_at',
'slug',
('deleted', '-created_at'),
] + db.Owned.meta['indexes'],
'ordering': ['-created_at'],
'queryset_class':
HarvestSourceQuerySet,
}
def __unicode__(self):
return self.name or ''
class WithMetrics(object):
metrics = db.DictField()
def clean(self):
'''Fill metrics with defaults on create'''
if not self.metrics:
self.metrics = dict((name, spec.default) for name, spec in (
metric_catalog.get(self.__class__, {}).items()))
return super(WithMetrics, self).clean()
class PiwikTracking(db.Document):
url = db.StringField(required=True)
date = db.DateTimeField(required=True, default=datetime.now)
kwargs = db.DictField()
meta = {
'indexes': ['date'],
'ordering': ['date'],
}
class OrgUnit(object):
'''
Simple mixin holding common fields for all organization units.
'''
name = db.StringField(max_length=255, required=True)
slug = db.SlugField(max_length=255, required=True, populate_from='name', update=True)
description = db.StringField(required=True)
url = db.URLField(max_length=255)
image_url = db.URLField(max_length=255)
extras = db.DictField()
class Fake(db.Document):
title = db.StringField()
description = db.StringField()
tags = db.ListField(db.StringField())
other = db.ListField(db.StringField())
nested = db.ListField(db.EmbeddedDocumentField(NestedFake))
metrics = db.DictField()
def __unicode__(self):
return 'fake'
class HarvestItem(db.EmbeddedDocument):
remote_id = db.StringField()
dataset = db.ReferenceField(Dataset)
status = db.StringField(choices=HARVEST_ITEM_STATUS.keys(),
default=DEFAULT_HARVEST_ITEM_STATUS, required=True)
created = db.DateTimeField(default=datetime.now, required=True)
started = db.DateTimeField()
ended = db.DateTimeField()
errors = db.ListField(db.EmbeddedDocumentField(HarvestError))
args = db.ListField(db.StringField())
kwargs = db.DictField()
class GeoZone(db.Document):
id = db.StringField(primary_key=True)
name = db.StringField(required=True)
level = db.StringField(required=True)
code = db.StringField(unique_with='level')
geom = db.MultiPolygonField(required=True)
parents = db.ListField()
keys = db.DictField()
population = db.IntField()
area = db.FloatField()
meta = {
'indexes': [
'name',
'parents',
('level', 'code'),
]
}
def __unicode__(self):
return self.id
__str__ = __unicode__
def __html__(self):
return gettext(self.name) + ' <i>(' + self.code + ')</i>'
@property
def keys_values(self):
"""Key values might be a list or not, always return a list."""
keys_values = []
for value in self.keys.values():
if isinstance(value, list):
keys_values += value
elif not str(value).startswith('-'): # Avoid -99.
keys_values.append(value)
return keys_values
def toGeoJSON(self):
return {
'id': self.id,
'type': 'Feature',
'geometry': self.geom,
'properties': {
'name': self.name,
'level': self.level,
'code': self.code,
'parents': self.parents,
'keys': self.keys,
'population': self.population,
'area': self.area,
}
}
class Tag(db.Document):
name = db.StringField(required=True, unique=True)
counts = db.DictField()
total = db.IntField(default=0)
meta = {
'indexes': ['name', '-total'],
'ordering': [
'-total',
],
}
def clean(self):
self.total = sum(self.counts.values())
class Tag(db.Document):
'''
This collection is auto-populated every hour map-reducing tag properties
from Datasets dans Reuses.
'''
name = db.StringField(required=True, unique=True)
counts = db.DictField()
total = db.IntField(default=0)
meta = {
'indexes': ['name', '-total'],
'ordering': ['-total', ],
}
def clean(self):
self.total = sum(self.counts.values())
class HarvestJob(db.Document):
'''Keep track of harvestings'''
created = db.DateTimeField(default=datetime.now, required=True)
started = db.DateTimeField()
ended = db.DateTimeField()
status = db.StringField(choices=HARVEST_JOB_STATUS.keys(),
default=DEFAULT_HARVEST_JOB_STATUS,
required=True)
errors = db.ListField(db.EmbeddedDocumentField(HarvestError))
items = db.ListField(db.EmbeddedDocumentField(HarvestItem))
source = db.ReferenceField(HarvestSource, reverse_delete_rule=db.CASCADE)
data = db.DictField()
meta = {
'indexes': ['-created', 'source', ('source', '-created')],
'ordering': ['-created'],
}
class Fake(db.Document):
title = db.StringField()
description = db.StringField()
tags = db.ListField(db.StringField())
other = db.ListField(db.StringField())
nested = db.ListField(db.EmbeddedDocumentField(NestedFake))
sub = db.EmbeddedDocumentField(NestedFake)
metrics = db.DictField()
__metrics_keys__ = [
'fake-metric-int',
'fake-metric-float',
]
def __str__(self):
return 'fake'
def get_metrics(self):
return self.metrics
class Activity(db.Document):
'''Store the activity entries for a single related object'''
actor = db.ReferenceField('User', required=True)
organization = db.ReferenceField('Organization')
related_to = db.ReferenceField(db.DomainModel, required=True)
created_at = db.DateTimeField(default=datetime.now, required=True)
kwargs = db.DictField()
on_new = Signal()
__metaclass__ = EmitNewActivityMetaClass
meta = {
'indexes': [
'actor',
'organization',
'related_to',
'-created_at',
('actor', '-created_at'),
('organization', '-created_at'),
('related_to', '-created_at'),
],
'allow_inheritance': True,
}
key = None
label = None
badge_type = 'primary'
icon = 'fa fa-info-circle'
template = 'activity/base.html'
@classmethod
def connect(cls, func):
return cls.on_new.connect(func, sender=cls)
@classmethod
def emit(cls, related_to, organization=None, **kwargs):
return cls.objects.create(
actor=current_user._get_current_object(),
related_to=related_to,
organization=organization
)
class GeoZone(db.Document):
id = db.StringField(primary_key=True)
name = db.StringField(required=True)
level = db.StringField(required=True)
code = db.StringField(unique_with='level')
geom = db.MultiPolygonField(required=True)
parents = db.ListField()
keys = db.DictField()
population = db.IntField()
area = db.FloatField()
meta = {
'indexes': [
'name',
'parents',
('level', 'code'),
]
}
def __unicode__(self):
return self.id
__str__ = __unicode__
def __html__(self):
return gettext(self.name) + ' <i>(' + self.code + ')</i>'
def toGeoJSON(self):
return {
'id': self.id,
'type': 'Feature',
'geometry': self.geom,
'properties': {
'name': self.name,
'level': self.level,
'code': self.code,
'parents': self.parents,
'keys': self.keys,
'population': self.population,
'area': self.area,
}
}
class Territory(db.Document):
name = db.StringField(required=True)
level = db.StringField(required=True)
code = db.StringField(unique_with='level')
geom = db.MultiPolygonField(required=True)
keys = db.DictField()
meta = {
'indexes': [
'name',
('level', 'code'),
]
}
def reference(self):
return TerritoryReference(id=self.id, name=self.name, level=self.level, code=self.code)
def __unicode__(self):
return self.name
class Site(WithMetrics, db.Document):
id = db.StringField(primary_key=True)
title = db.StringField(required=True)
keywords = db.ListField(db.StringField())
feed_size = db.IntField(required=True, default=DEFAULT_FEED_SIZE)
configs = db.DictField()
themes = db.DictField()
settings = db.EmbeddedDocumentField(SiteSettings, default=SiteSettings)
__metrics_keys__ = [
'max_dataset_followers',
'max_dataset_reuses',
'max_reuse_datasets',
'max_reuse_followers',
'max_org_followers',
'max_org_reuses',
'max_org_datasets',
'datasets',
'discussions',
'followers',
'organizations',
'public-service',
'resources',
'reuses',
'users',
]
def __str__(self):
return self.title or ''
def count_users(self):
from udata.models import User
self.metrics['users'] = User.objects(confirmed_at__ne=None,
deleted=None).count()
self.save()
def count_org(self):
from udata.models import Organization
self.metrics['organizations'] = Organization.objects.visible().count()
self.save()
def count_org_for_badge(self, badge_kind):
from udata.models import Organization
self.metrics[badge_kind] = Organization.objects(
badges__kind=badge_kind).count()
self.save()
def count_datasets(self):
from udata.models import Dataset
self.metrics['datasets'] = Dataset.objects.visible().count()
self.save()
def count_resources(self):
self.metrics['resources'] = next(
Dataset.objects.visible().aggregate(
{'$project': {
'resources': 1
}}, {'$unwind': '$resources'},
{'$group': {
'_id': 'result',
'count': {
'$sum': 1
}
}}), {}).get('count', 0)
self.save()
def count_reuses(self):
self.metrics['reuses'] = Reuse.objects.visible().count()
self.save()
def count_followers(self):
from udata.models import Follow
self.metrics['followers'] = Follow.objects(until=None).count()
self.save()
def count_discussions(self):
from udata.models import Discussion
self.metrics['discussions'] = Discussion.objects.count()
self.save()
def count_max_dataset_followers(self):
dataset = (Dataset.objects(metrics__followers__gt=0).visible().
order_by('-metrics.followers').first())
self.metrics['max_dataset_followers'] = dataset.metrics[
'followers'] if dataset else 0
self.save()
def count_max_dataset_reuses(self):
dataset = (Dataset.objects(metrics__reuses__gt=0).visible().order_by(
'-metrics.reuses').first())
self.metrics[
'max_dataset_reuses'] = dataset.metrics['reuses'] if dataset else 0
self.save()
def count_max_reuse_datasets(self):
reuse = (Reuse.objects(metrics__datasets__gt=0).visible().order_by(
'-metrics.datasets').first())
self.metrics[
'max_reuse_datasets'] = reuse.metrics['datasets'] if reuse else 0
self.save()
def count_max_reuse_followers(self):
reuse = (Reuse.objects(metrics__followers__gt=0).visible().order_by(
'-metrics.followers').first())
self.metrics[
'max_reuse_followers'] = reuse.metrics['followers'] if reuse else 0
self.save()
def count_max_org_followers(self):
org = (Organization.objects(metrics__followers__gt=0).visible().
order_by('-metrics.followers').first())
self.metrics[
'max_org_followers'] = org.metrics['followers'] if org else 0
self.save()
def count_max_org_reuses(self):
org = (Organization.objects(metrics__reuses__gt=0).visible().order_by(
'-metrics.reuses').first())
self.metrics['max_org_reuses'] = org.metrics['reuses'] if org else 0
self.save()
def count_max_org_datasets(self):
org = (Organization.objects(metrics__datasets__gt=0).visible().
order_by('-metrics.datasets').first())
self.metrics[
'max_org_datasets'] = org.metrics['datasets'] if org else 0
self.save()
class GeoZone(db.Document):
id = db.StringField(primary_key=True)
slug = db.StringField(required=True)
name = db.StringField(required=True)
level = db.StringField(required=True)
code = db.StringField(required=True)
geom = db.MultiPolygonField()
parents = db.ListField()
keys = db.DictField()
validity = db.EmbeddedDocumentField(db.DateRange)
ancestors = db.ListField()
successors = db.ListField()
population = db.IntField()
area = db.FloatField()
wikipedia = db.StringField()
dbpedia = db.StringField()
flag = db.ImageField(fs=logos)
blazon = db.ImageField(fs=logos)
logo = db.ImageField(fs=logos)
meta = {
'indexes': [
'name',
'parents',
('level', 'code'),
],
'queryset_class': GeoZoneQuerySet
}
def __unicode__(self):
return self.id
__str__ = __unicode__
def __html__(self):
"""In use within the admin."""
return '{name} <i>({code})</i>'.format(name=gettext(self.name),
code=self.code)
def logo_url(self, external=False):
flag_filename = self.flag.filename
blazon_filename = self.blazon.filename
if flag_filename and self.flag.fs.exists(flag_filename):
return self.flag.fs.url(flag_filename, external=external)
elif blazon_filename and self.blazon.fs.exists(blazon_filename):
return self.blazon.fs.url(blazon_filename, external=external)
else:
return ''
@property
def keys_values(self):
"""Key values might be a list or not, always return a list."""
keys_values = []
for value in self.keys.values():
if isinstance(value, list):
keys_values += value
elif not str(value).startswith('-'): # Avoid -99.
keys_values.append(value)
return keys_values
@cached_property
def level_code(self):
"""Truncated level code for the sake of readability."""
# Either 'region', 'departement' or 'commune',
# useful to match TERRITORY_DATASETS keys.
return self.id.split(':')[1]
@cached_property
def level_name(self):
"""Truncated level name for the sake of readability."""
if self.level.startswith('fr:'):
return self.level[3:]
# Keep the whole level name as a fallback (e.g. `country:fr`)
return self.level
@cached_property
def level_i18n_name(self):
"""In use within templates for dynamic translations."""
for level, name in spatial_granularities:
if self.level == level:
return name
return self.level_name # Fallback that should never happen.
@cached_property
def ancestors_objects(self):
"""Ancestors objects sorted by name."""
ancestors_objects = []
for ancestor in self.ancestors:
try:
ancestor_object = GeoZone.objects.get(id=ancestor)
except GeoZone.DoesNotExist:
continue
ancestors_objects.append(ancestor_object)
ancestors_objects.sort(key=lambda a: a.name)
return ancestors_objects
@cached_property
def child_level(self):
"""Return the child level given handled levels."""
HANDLED_LEVELS = current_app.config.get('HANDLED_LEVELS')
try:
return HANDLED_LEVELS[HANDLED_LEVELS.index(self.level) - 1]
except (IndexError, ValueError):
return None
@cached_property
def parent_level(self):
"""Return the parent level given handled levels."""
HANDLED_LEVELS = current_app.config.get('HANDLED_LEVELS')
try:
return HANDLED_LEVELS[HANDLED_LEVELS.index(self.level) + 1]
except (IndexError, ValueError):
return None
@property
def url(self):
return url_for('territories.territory', territory=self)
@property
def external_url(self):
return url_for('territories.territory', territory=self, _external=True)
@cached_property
def wikipedia_url(self):
"""Computed wikipedia URL from the DBpedia one."""
return (self.dbpedia.replace('dbpedia',
'wikipedia').replace('resource', 'wiki'))
@cached_property
def postal_string(self):
"""Return a list of postal codes separated by commas."""
return ', '.join(self.keys.get('postal', []))
@property
def parents_objects(self):
if self.parent_level:
for parent in self.parents:
if parent.startswith(self.parent_level):
yield GeoZone.objects.get(id=parent,
level=self.parent_level)
@cached_property
def current_parent(self):
today = date.today()
for parent in self.parents_objects:
if parent.valid_at(today):
return parent
@property
def children(self):
return (GeoZone.objects(level=self.child_level,
parents__in=[self.id]).order_by('name'))
@property
def biggest_children(self):
return self.children.order_by('-population', '-area')[:10]
@property
def handled_level(self):
return self.level in current_app.config.get('HANDLED_LEVELS')
def valid_at(self, valid_date):
if not self.validity:
return True
return self.validity.start <= valid_date <= self.validity.end
def toGeoJSON(self):
return {
'id': self.id,
'type': 'Feature',
'geometry': self.geom,
'properties': {
'slug': self.slug,
'name': gettext(self.name),
'level': self.level,
'code': self.code,
'validity': self.validity,
'parents': self.parents,
'keys': self.keys,
'population': self.population,
'area': self.area,
'logo': self.logo_url(external=True)
}
}
class GeoZone(db.Document):
id = db.StringField(primary_key=True)
name = db.StringField(required=True)
level = db.StringField(required=True)
code = db.StringField(unique_with='level')
geom = db.MultiPolygonField(required=True)
parents = db.ListField()
keys = db.DictField()
population = db.IntField()
area = db.FloatField()
wikipedia = db.StringField()
dbpedia = db.StringField()
logo = db.ImageField(fs=logos)
meta = {
'indexes': [
'name',
'parents',
('level', 'code'),
]
}
def __unicode__(self):
return self.id
__str__ = __unicode__
def __html__(self):
return '{name} <i>({code})</i>'.format(
name=gettext(self.name), code=self.code)
def logo_url(self, external=False):
filename = self.logo.filename
if filename and self.logo.fs.exists(filename):
return self.logo.fs.url(filename, external=external)
else:
return ''
@property
def keys_values(self):
"""Key values might be a list or not, always return a list."""
keys_values = []
for value in self.keys.values():
if isinstance(value, list):
keys_values += value
elif not str(value).startswith('-'): # Avoid -99.
keys_values.append(value)
return keys_values
@property
def url(self):
return url_for('territories.territory', territory=self)
@property
def external_url(self):
return url_for('territories.territory', territory=self, _external=True)
@cached_property
def wikipedia_url(self):
return (self.dbpedia.replace('dbpedia', 'wikipedia')
.replace('resource', 'wiki'))
@cached_property
def postal_string(self):
"""Return a list of postal codes separated by commas."""
return ', '.join(self.keys.get('postal', []))
@cached_property
def town_repr(self):
"""Representation of a town with optional county."""
if self.county:
return '{name} <small>({county_name})</small>'.format(
name=self.name, county_name=self.county.name)
return self.name
@cached_property
def county(self):
for parent in self.parents:
if parent.startswith('fr/county'):
return GeoZone.objects.get(id=parent)
def toGeoJSON(self):
return {
'id': self.id,
'type': 'Feature',
'geometry': self.geom,
'properties': {
'name': _(self.name),
'level': self.level,
'code': self.code,
'parents': self.parents,
'keys': self.keys,
'population': self.population,
'area': self.area,
}
}
class GeoZone(db.Document):
id = db.StringField(primary_key=True)
name = db.StringField(required=True)
level = db.StringField(required=True)
code = db.StringField(unique_with='level')
geom = db.MultiPolygonField(required=True)
parents = db.ListField()
keys = db.DictField()
population = db.IntField()
area = db.FloatField()
wikipedia = db.StringField()
dbpedia = db.StringField()
logo = db.ImageField(fs=logos)
meta = {
'indexes': [
'name',
'parents',
('level', 'code'),
]
}
def __unicode__(self):
return self.id
__str__ = __unicode__
def __html__(self):
"""In use within the admin."""
return '{name} <i>({code})</i>'.format(name=gettext(self.name),
code=self.code)
@cached_property
def html_title(self):
"""In use within templates."""
if self.level_name == 'town':
return ('{name} '
'<small>(<a href="{parent_url}">{parent_name}</a>)</small>'
'').format(name=self.name,
parent_url=self.parent.url,
parent_name=self.parent.name)
elif self.level_name == 'county':
return '{name} <small>({code})</small>'.format(name=self.name,
code=self.code)
else:
return self.name
def logo_url(self, external=False):
filename = self.logo.filename
if filename and self.logo.fs.exists(filename):
return self.logo.fs.url(filename, external=external)
else:
return ''
@property
def keys_values(self):
"""Key values might be a list or not, always return a list."""
keys_values = []
for value in self.keys.values():
if isinstance(value, list):
keys_values += value
elif not str(value).startswith('-'): # Avoid -99.
keys_values.append(value)
return keys_values
@cached_property
def level_name(self):
"""Truncated level name for the sake of readability."""
if self.level.startswith('fr/'):
return self.level[3:]
# Keep the whole level name as a fallback (e.g. `country/fr`)
return self.level
@cached_property
def level_i18n_name(self):
"""In use within templates for dynamic translations."""
for level, name in spatial_granularities:
if self.level == level:
return name
return self.level_name # Fallback that should never happen.
@cached_property
def child_level(self):
"""Return the child level given handled levels."""
HANDLED_LEVELS = current_app.config.get('HANDLED_LEVELS')
try:
return HANDLED_LEVELS[HANDLED_LEVELS.index(self.level) - 1]
except (IndexError, ValueError):
return None
@cached_property
def parent_level(self):
"""Return the parent level given handled levels."""
HANDLED_LEVELS = current_app.config.get('HANDLED_LEVELS')
try:
return HANDLED_LEVELS[HANDLED_LEVELS.index(self.level) + 1]
except (IndexError, ValueError):
return None
@property
def url(self):
return url_for('territories.territory', territory=self)
@property
def external_url(self):
return url_for('territories.territory', territory=self, _external=True)
@cached_property
def wikipedia_url(self):
"""Computed wikipedia URL from the DBpedia one."""
return (self.dbpedia.replace('dbpedia',
'wikipedia').replace('resource', 'wiki'))
@cached_property
def postal_string(self):
"""Return a list of postal codes separated by commas."""
return ', '.join(self.keys.get('postal', []))
@property
def parent(self):
if self.parent_level:
for parent in self.parents:
if parent.startswith(self.parent_level):
return GeoZone.objects.get(id=parent,
level=self.parent_level)
@property
def children(self):
if self.child_level:
return (GeoZone.objects(level=self.child_level,
parents__in=[self.id]).order_by(
'-population', '-area'))
@property
def handled_level(self):
return self.level in current_app.config.get('HANDLED_LEVELS')
def toGeoJSON(self):
return {
'id': self.id,
'type': 'Feature',
'geometry': self.geom,
'properties': {
'name': gettext(self.name),
'level': self.level,
'code': self.code,
'parents': self.parents,
'keys': self.keys,
'population': self.population,
'area': self.area,
'logo': self.logo_url(external=True)
}
}
class Fake(db.Document):
raw = db.DictField()
class ResourceMixin(object):
id = db.AutoUUIDField(primary_key=True)
title = db.StringField(verbose_name="Title", required=True)
description = db.StringField()
filetype = db.StringField(choices=list(RESOURCE_FILETYPES),
default='file',
required=True)
type = db.StringField(choices=list(RESOURCE_TYPES),
default='main',
required=True)
url = db.URLField(required=True)
urlhash = db.StringField()
checksum = db.EmbeddedDocumentField(Checksum)
format = db.StringField()
mime = db.StringField()
filesize = db.IntField() # `size` is a reserved keyword for mongoengine.
fs_filename = db.StringField()
extras = db.ExtrasField()
schema = db.DictField()
created_at = db.DateTimeField(default=datetime.now, required=True)
modified = db.DateTimeField(default=datetime.now, required=True)
published = db.DateTimeField(default=datetime.now, required=True)
deleted = db.DateTimeField()
def clean(self):
super(ResourceMixin, self).clean()
if not self.urlhash or 'url' in self._get_changed_fields():
self.urlhash = hash_url(self.url)
@cached_property # Accessed at least 2 times in front rendering
def preview_url(self):
return get_preview_url(self)
@property
def closed_or_no_format(self):
"""
Return True if the specified format is in CLOSED_FORMATS or
no format has been specified.
"""
return not self.format or self.format.lower() in CLOSED_FORMATS
def check_availability(self):
'''
Return the check status from extras if any.
NB: `unknown` will evaluate to True in the aggregate checks using
`all([])` (dataset, organization, user).
'''
return self.extras.get('check:available', 'unknown')
def need_check(self):
'''Does the resource needs to be checked against its linkchecker?
We check unavailable resources often, unless they go over the
threshold. Available resources are checked less and less frequently
based on their historical availability.
'''
min_cache_duration, max_cache_duration, ko_threshold = [
current_app.config.get(k) for k in (
'LINKCHECKING_MIN_CACHE_DURATION',
'LINKCHECKING_MAX_CACHE_DURATION',
'LINKCHECKING_UNAVAILABLE_THRESHOLD',
)
]
count_availability = self.extras.get('check:count-availability', 1)
is_available = self.check_availability()
if is_available == 'unknown':
return True
elif is_available or count_availability > ko_threshold:
delta = min(min_cache_duration * count_availability,
max_cache_duration)
else:
delta = min_cache_duration
if self.extras.get('check:date'):
limit_date = datetime.now() - timedelta(minutes=delta)
check_date = self.extras['check:date']
if not isinstance(check_date, datetime):
try:
check_date = parse_dt(check_date)
except (ValueError, TypeError):
return True
if check_date >= limit_date:
return False
return True
@property
def latest(self):
'''
Permanent link to the latest version of this resource.
If this resource is updated and `url` changes, this property won't.
'''
return endpoint_for('datasets.resource',
'api.resource_redirect',
id=self.id,
_external=True)
@cached_property
def json_ld(self):
result = {
'@type': 'DataDownload',
'@id': str(self.id),
'url': self.latest,
'name': self.title or _('Nameless resource'),
'contentUrl': self.url,
'dateCreated': self.created_at.isoformat(),
'dateModified': self.modified.isoformat(),
'datePublished': self.published.isoformat(),
'extras':
[get_json_ld_extra(*item) for item in self.extras.items()],
}
if 'views' in self.metrics:
result['interactionStatistic'] = {
'@type': 'InteractionCounter',
'interactionType': {
'@type': 'DownloadAction',
},
'userInteractionCount': self.metrics['views']
}
if self.format:
result['encodingFormat'] = self.format
if self.filesize:
result['contentSize'] = self.filesize
if self.mime:
result['fileFormat'] = self.mime
if self.description:
result['description'] = mdstrip(self.description)
return result
class Nested(db.EmbeddedDocument):
id = db.AutoUUIDField()
name = db.StringField()
sub = db.EmbeddedDocumentField(SubNested)
raw = db.DictField()