def get_context(self):
context = super(OrganizationDetailView, self).get_context()
can_edit = EditOrganizationPermission(self.organization)
can_view = OrganizationPrivatePermission(self.organization)
if self.organization.deleted and not can_view.can():
abort(410)
datasets = Dataset.objects(organization=self.organization).order_by('-temporal_coverage.end', '-metrics.reuses', '-metrics.followers').visible()
reuses = Reuse.objects(organization=self.organization).order_by('-metrics.reuses', '-metrics.followers').visible()
followers = (Follow.objects.followers(self.organization)
.order_by('follower.fullname'))
context.update({
'reuses': reuses.paginate(1, self.page_size),
'datasets': datasets.paginate(1, self.page_size),
'followers': followers,
'can_edit': can_edit,
'can_view': can_view,
'private_reuses': (
list(Reuse.objects(organization=self.object).hidden())
if can_view else []),
'private_datasets': (
list(Dataset.objects(organization=self.object).hidden())
if can_view else []),
})
return context
def get_context(self):
context = super(OrganizationDetailView, self).get_context()
can_edit = EditOrganizationPermission(self.organization)
can_view = OrganizationPrivatePermission(self.organization)
if self.organization.deleted and not can_view.can():
abort(410)
datasets = Dataset.objects(organization=self.organization).order_by(
'-temporal_coverage.end', '-metrics.reuses',
'-metrics.followers').visible()
reuses = Reuse.objects(organization=self.organization).order_by(
'-metrics.reuses', '-metrics.followers').visible()
followers = (Follow.objects.followers(
self.organization).order_by('follower.fullname'))
context.update({
'reuses':
reuses.paginate(1, self.page_size),
'datasets':
datasets.paginate(1, self.page_size),
'followers':
followers,
'can_edit':
can_edit,
'can_view':
can_view,
'private_reuses':
(list(Reuse.objects(
organization=self.object).hidden()) if can_view else []),
'private_datasets':
(list(Dataset.objects(
organization=self.object).hidden()) if can_view else []),
})
return context
def remote_reuses(self):
# dataset_ids = (d.ext['harvest'].remote_id for d in Dataset.objects(ext__harvest__harvester=self.harvester.id))
# response = self.get('package_list')
# for dataset_id in response['result']:
for dataset in Dataset.objects(
ext__harvest__harvester=self.harvester.id).timeout(False):
try:
resp = self.get('related_list',
{'id': dataset.ext['harvest'].remote_id})
except:
log.error('Unable to parse reuse for dataset %s', dataset.id)
continue
for details in resp['result']:
reuse_url = details['url']
urlhash = Reuse.hash_url(reuse_url)
reuse, _ = Reuse.objects.get_or_create(urlhash=urlhash,
auto_save=False)
reuse.url = reuse_url
reuse.title = details['title']
reuse.description = details['description']
reuse.type = details['type']
# reuse.url = details['url']
reuse.image_url = details.get('image_url')
reuse.featured = bool(details.get('featured', False))
reuse.created_at = parse(details['created'])
if details.get('owner_id'):
reuse.owner = self.get_harvested(User, details['owner_id'])
if not dataset in reuse.datasets:
reuse.datasets.append(dataset)
for tag in dataset.tags:
if not tag in reuse.tags:
reuse.tags.append(tag)
yield reuse
def get_context(self):
context = super(OrganizationIssuesView, self).get_context()
datasets = Dataset.objects(organization=self.organization)
reuses = Reuse.objects(organization=self.organization)
ids = [o.id for o in list(datasets) + list(reuses)]
context['issues'] = Issue.objects(subject__in=ids)
return context
def get_context(self):
context = super(OrganizationDetailView, self).get_context()
org_id = str(self.organization.id)
datasets, supplied_datasets, reuses = search.multiquery(
search.SearchQuery(Dataset, sort='-created', organization=org_id, page_size=9),
search.SearchQuery(Dataset, sort='-created', supplier=org_id, page_size=9),
search.SearchQuery(Reuse, sort='-created', organization=org_id, page_size=9),
)
followers = FollowOrg.objects.followers(self.organization).order_by('follower.fullname')
can_edit = EditOrganizationPermission(self.organization.id)
context.update({
'reuses': reuses,
'datasets': datasets,
'supplied_datasets': supplied_datasets,
'followers': followers[:self.nb_followers],
'can_edit': can_edit
})
if can_edit:
context.update({
'private_reuses': list(Reuse.objects(organization=self.object, private=True)),
'private_datasets': list(Dataset.objects(organization=self.object, private=True)),
})
return context
def test_purge_reuses(self):
reuse = ReuseFactory(title='test-reuse')
# Upload reuse's image
file = create_test_image()
user = AdminFactory()
self.login(user)
response = self.post(url_for('api.reuse_image', reuse=reuse),
{'file': (file, 'test.png')},
json=False)
self.assert200(response)
# Delete reuse
response = self.delete(url_for('api.reuse', reuse=reuse))
self.assert204(response)
user = UserFactory()
transfer = Transfer.objects.create(
owner=user,
recipient=user,
subject=reuse,
comment='comment',
)
tasks.purge_reuses()
assert Transfer.objects.filter(id=transfer.id).count() == 0
# Check reuse's image is deleted
self.assertEqual(list(storages.images.list_files()), [])
deleted_reuse = Reuse.objects(title='test-reuse').first()
self.assertIsNone(deleted_reuse)
def aggregate_reuses_daily(org, day):
keys = ['reuses_{0}'.format(k) for k in KEYS]
ids = [r.id for r in Reuse.objects(organization=org).only('id')]
metrics = Metrics.objects(object_id__in=ids,
level='daily', date=day.isoformat())
values = [int(metrics.sum('values.{0}'.format(k))) for k in KEYS]
Metrics.objects.update_daily(org, day, **dict(zip(keys, values)))
def remote_reuses(self):
# dataset_ids = (d.ext['harvest'].remote_id for d in Dataset.objects(ext__harvest__harvester=self.harvester.id))
# response = self.get('package_list')
# for dataset_id in response['result']:
for dataset in Dataset.objects(ext__harvest__harvester=self.harvester.id).timeout(False):
try:
resp = self.get('related_list', {'id': dataset.ext['harvest'].remote_id})
except:
log.error('Unable to parse reuse for dataset %s', dataset.id)
continue
for details in resp['result']:
reuse_url = details['url']
urlhash = Reuse.hash_url(reuse_url)
reuse, _ = Reuse.objects.get_or_create(urlhash=urlhash, auto_save=False)
reuse.url = reuse_url
reuse.title = details['title']
reuse.description = details['description']
reuse.type = details['type']
# reuse.url = details['url']
reuse.image_url = details.get('image_url')
reuse.featured = bool(details.get('featured', False))
reuse.created_at = parse(details['created'])
if details.get('owner_id'):
reuse.owner = self.get_harvested(User, details['owner_id'])
if not dataset in reuse.datasets:
reuse.datasets.append(dataset)
for tag in dataset.tags:
if not tag in reuse.tags:
reuse.tags.append(tag)
yield reuse
def get_context(self):
context = super(DatasetDetailView, self).get_context()
if self.dataset.private and not DatasetEditPermission(self.dataset).can():
abort(404)
context['reuses'] = Reuse.objects(datasets=self.dataset)
context['can_edit'] = DatasetEditPermission(self.dataset)
context['can_edit_resource'] = CommunityResourceEditPermission
return context
def get_value(self):
ids = [
d.id
for d in (Reuse.objects(organization=self.target).only('id') or [])
]
return int(
Metrics.objects(object_id__in=ids,
level='daily').sum('values.nb_uniq_visitors'))
def aggregate_reuses_daily(org, day):
keys = ['reuses_{0}'.format(k) for k in KEYS]
ids = [r.id for r in Reuse.objects(organization=org).only('id')]
metrics = Metrics.objects(object_id__in=ids,
level='daily',
date=day.isoformat())
values = [int(metrics.sum('values.{0}'.format(k))) for k in KEYS]
Metrics.objects.update_daily(org, day, **dict(zip(keys, values)))
def migrate(db):
log.info('Processing reuses.')
reuses = Reuse.objects().no_cache().timeout(False)
for reuse in reuses:
reuse.count_datasets()
log.info('Completed.')
def home():
context = {
'recent_datasets': Dataset.objects.visible(),
'recent_reuses': Reuse.objects(featured=True).visible(),
'last_post': Post.objects(private=False).first(),
}
processor = theme.current.get_processor('home')
context = processor(context)
return theme.render('home.html', **context)
def home():
context = {
'recent_datasets': Dataset.objects.visible(),
'recent_reuses': Reuse.objects(featured=True).visible(),
'last_post': Post.objects(private=False).first(),
}
processor = theme.current.get_processor('home')
context = processor(context)
return theme.render('home.html', **context)
def get_context(self):
context = super(DatasetDetailView, self).get_context()
if self.dataset.private and not DatasetEditPermission(
self.dataset).can():
abort(404)
context['reuses'] = Reuse.objects(datasets=self.dataset)
context['can_edit'] = DatasetEditPermission(self.dataset)
context['can_edit_resource'] = CommunityResourceEditPermission
return context
def dataconnexions5():
reuses = Reuse.objects(badges__kind=DATACONNEXIONS_5_CANDIDATE).visible()
categories = [{
'tag': tag,
'label': label,
'description': description,
'reuses': reuses(tags=tag),
} for tag, label, description in DATACONNEXIONS_5_CATEGORIES]
return theme.render('dataconnexions-5.html', categories=categories)
def dataconnexions5():
reuses = Reuse.objects(badges__kind=DATACONNEXIONS_5_CANDIDATE).visible()
categories = [{
'tag': tag,
'label': label,
'description': description,
'reuses': reuses(tags=tag),
} for tag, label, description in DATACONNEXIONS_5_CATEGORIES]
return theme.render('dataconnexions-5.html', categories=categories)
def dataconnexions6():
# Use tags until we are sure all reuse are correctly labeled
# reuses = Reuse.objects(badges__kind=DATACONNEXIONS_6_CANDIDATE)
reuses = Reuse.objects(tags='dataconnexions-6').visible()
categories = [{
'tag': tag,
'label': label,
'description': description,
'reuses': reuses(tags=tag),
} for tag, label, description in DATACONNEXIONS_6_CATEGORIES]
return theme.render('dataconnexions-6.html', categories=categories)
def explore():
recent_datasets = list(Dataset.objects.visible().order_by('-date').limit(9))
recent_reuses = list(Reuse.objects.order_by('-date').limit(9))
featured_datasets = list(Dataset.objects(featured=True).visible().order_by('-date').limit(15))
featured_reuses = list(Reuse.objects(featured=True).order_by('-date').limit(15))
return render('explore.html',
recent_datasets=recent_datasets,
recent_reuses=recent_reuses,
featured_datasets=featured_datasets,
featured_reuses=featured_reuses,
)
def migrate(db):
log.info('Processing Reuse.')
reuses = Reuse.objects().no_cache().timeout(False)
count = 0
for reuse in reuses:
reuse.topic = 'others'
reuse.save()
count += 1
log.info(f'Modified {count} Reuses objects')
log.info('Done')
def get_context(self):
context = super(DatasetDetailView, self).get_context()
if not DatasetEditPermission(self.dataset).can():
if self.dataset.private:
abort(404)
elif self.dataset.deleted:
abort(410)
context['reuses'] = Reuse.objects(datasets=self.dataset).visible()
context['can_edit'] = DatasetEditPermission(self.dataset)
context['can_edit_resource'] = ResourceEditPermission
return context
def dataconnexions6():
# Use tags until we are sure all reuse are correctly labeled
# reuses = Reuse.objects(badges__kind=DATACONNEXIONS_6_CANDIDATE)
reuses = Reuse.objects(tags='dataconnexions-6').visible()
categories = [{
'tag': tag,
'label': label,
'description': description,
'reuses': reuses(tags=tag),
} for tag, label, description in DATACONNEXIONS_6_CATEGORIES]
return theme.render('dataconnexions-6.html', categories=categories)
def purge_reuses(self):
for reuse in Reuse.objects(deleted__ne=None):
log.info('Purging reuse "{0}"'.format(reuse))
# Remove followers
FollowReuse.objects(following=reuse).delete()
# Remove issues
ReuseIssue.objects(subject=reuse).delete()
# Remove activity
Activity.objects(related_to=reuse).delete()
# Remove metrics
Metrics.objects(object_id=reuse.id).delete()
reuse.delete()
def get_context(self):
context = super(DatasetDetailView, self).get_context()
if not DatasetEditPermission(self.dataset).can():
if self.dataset.private:
abort(404)
elif self.dataset.deleted:
abort(410)
context['reuses'] = Reuse.objects(datasets=self.dataset).visible()
context['can_edit'] = DatasetEditPermission(self.dataset)
context['can_edit_resource'] = ResourceEditPermission
context['discussions'] = DatasetDiscussion.objects(
subject=self.dataset)
return context
def get_context(self):
context = super(OrganizationDetailView, self).get_context()
datasets = Dataset.objects(organization=self.organization).visible().order_by('-created')
supplied_datasets = Dataset.objects(supplier=self.organization).visible().order_by('-created')
reuses = Reuse.objects(organization=self.organization).visible().order_by('-created')
followers = FollowOrg.objects.followers(self.organization).order_by('follower.fullname')
can_edit = EditOrganizationPermission(self.organization)
can_view = OrganizationPrivatePermission(self.organization)
context.update({
'reuses': reuses.paginate(1, self.page_size),
'datasets': datasets.paginate(1, self.page_size),
'supplied_datasets': supplied_datasets[:self.page_size],
'followers': followers[:self.nb_followers],
'can_edit': can_edit,
'can_view': can_view,
'private_reuses': list(Reuse.objects(organization=self.object).hidden()) if can_view else [],
'private_datasets': list(Dataset.objects(organization=self.object).hidden()) if can_view else [],
})
return context
def get_context(self):
context = super(DatasetDetailView, self).get_context()
if (self.dataset.private
and not DatasetEditPermission(self.dataset).can()):
abort(404)
elif self.dataset.deleted:
abort(410)
context['reuses'] = Reuse.objects(datasets=self.dataset)
context['can_edit'] = DatasetEditPermission(self.dataset)
context['can_edit_resource'] = CommunityResourceEditPermission
context['discussions'] = DatasetDiscussion.objects(
subject=self.dataset)
return context
def get_context(self):
context = super(OrganizationDetailView, self).get_context()
datasets = Dataset.objects(
organization=self.organization).visible().order_by('-created')
supplied_datasets = Dataset.objects(
supplier=self.organization).visible().order_by('-created')
reuses = Reuse.objects(
organization=self.organization).visible().order_by('-created')
followers = FollowOrg.objects.followers(
self.organization).order_by('follower.fullname')
can_edit = EditOrganizationPermission(self.organization)
can_view = OrganizationPrivatePermission(self.organization)
context.update({
'reuses':
reuses.paginate(1, self.page_size),
'datasets':
datasets.paginate(1, self.page_size),
'supplied_datasets':
supplied_datasets[:self.page_size],
'followers':
followers[:self.nb_followers],
'can_edit':
can_edit,
'can_view':
can_view,
'private_reuses':
list(Reuse.objects(
organization=self.object).hidden()) if can_view else [],
'private_datasets':
list(Dataset.objects(
organization=self.object).hidden()) if can_view else [],
})
return context
def get_context(self):
context = super(OrganizationDetailView, self).get_context()
can_edit = EditOrganizationPermission(self.organization)
can_view = OrganizationPrivatePermission(self.organization)
if self.organization.deleted and not can_view.can():
abort(410)
datasets = Dataset.objects(organization=self.organization).visible()
reuses = Reuse.objects(organization=self.organization).visible()
followers = FollowOrg.objects.followers(self.organization).order_by("follower.fullname")
context.update(
{
"reuses": reuses.paginate(1, self.page_size),
"datasets": datasets.paginate(1, self.page_size),
"followers": followers,
"can_edit": can_edit,
"can_view": can_view,
"private_reuses": (list(Reuse.objects(organization=self.object).hidden()) if can_view else []),
"private_datasets": (list(Dataset.objects(organization=self.object).hidden()) if can_view else []),
}
)
return context
def get_context(self):
context = super(DatasetDetailView, self).get_context()
if not DatasetEditPermission(self.dataset).can():
if self.dataset.private:
abort(404)
elif self.dataset.deleted:
abort(410)
context['reuses'] = Reuse.objects(datasets=self.dataset).visible()
context['can_edit'] = DatasetEditPermission(self.dataset)
context['can_edit_resource'] = ResourceEditPermission
context['rdf_links'] = [(RDF_MIME_TYPES[fmt],
url_for('datasets.rdf_format',
dataset=self.dataset.id,
format=ext))
for (fmt, ext) in RDF_EXTENSIONS.items()]
return context
def get_context(self):
context = super(DatasetDetailView, self).get_context()
if not DatasetEditPermission(self.dataset).can():
if self.dataset.private:
abort(404)
elif self.dataset.deleted:
abort(410)
context['reuses'] = Reuse.objects(datasets=self.dataset).visible()
context['can_edit'] = DatasetEditPermission(self.dataset)
context['can_edit_resource'] = ResourceEditPermission
context['rdf_links'] = [
(RDF_MIME_TYPES[fmt],
url_for('datasets.rdf_format',
dataset=self.dataset.id, format=ext))
for (fmt, ext) in RDF_EXTENSIONS.items()
]
return context
def get_context(self):
context = super(OrganizationDetailView, self).get_context()
params_datasets_page = request.args.get('datasets_page', 1, type=int)
params_reuses_page = request.args.get('reuses_page', 1, type=int)
can_edit = EditOrganizationPermission(self.organization)
can_view = OrganizationPrivatePermission(self.organization)
if self.organization.deleted and not can_view.can():
abort(410)
datasets = Dataset.objects(organization=self.organization).order_by(
'-temporal_coverage.end', '-metrics.reuses', '-metrics.followers')
reuses = Reuse.objects(organization=self.organization).order_by(
'-metrics.reuses', '-metrics.followers')
followers = (Follow.objects.followers(
self.organization).order_by('follower.fullname'))
if not can_view:
datasets = datasets.visible()
reuses = reuses.visible()
context.update({
'reuses':
reuses.paginate(params_reuses_page, self.page_size),
'datasets':
datasets.paginate(params_datasets_page, self.page_size),
'total_datasets':
len(datasets),
'total_reuses':
len(reuses),
'followers':
followers,
'can_edit':
can_edit,
'can_view':
can_view,
})
return context
def get_context(self):
context = super(OrganizationDetailView, self).get_context()
org_id = str(self.organization.id)
datasets, supplied_datasets, reuses = multiquery(
SearchQuery(DatasetSearch, sort='-created', organization=org_id, page_size=9),
SearchQuery(DatasetSearch, sort='-created', supplier=org_id, page_size=9),
SearchQuery(ReuseSearch, sort='-created', organization=org_id, page_size=9),
)
context.update({
'reuses': reuses,
'datasets': datasets,
'supplied_datasets': supplied_datasets,
'private_reuses': list(Reuse.objects(organization=self.object, private=True)),
'private_datasets': list(Dataset.objects(organization=self.object, private=True)),
'can_edit': EditOrganizationPermission(self.organization.id)
})
return context
def test_purge_reuses(self):
reuse = ReuseFactory(title='test-reuse')
# Upload reuse's image
file = create_test_image()
user = AdminFactory()
self.login(user)
response = self.post(url_for('api.reuse_image', reuse=reuse),
{'file': (file, 'test.png')},
json=False)
self.assert200(response)
# Delete reuse
response = self.delete(url_for('api.reuse', reuse=reuse))
self.assert204(response)
tasks.purge_reuses()
# Check reuse's image is deleted
self.assertEqual(list(storages.images.list_files()), [])
organization = Reuse.objects(title='test-reuse').first()
self.assertIsNone(organization)
def get_value(self):
reuse = Reuse.objects(metrics__datasets__gt=0).visible().order_by('-metrics.datasets').first()
return reuse.metrics.get('datasets', 0)
def get_context(self):
context = super(UserReusesView, self).get_context()
context['reuses'] = Reuse.objects(owner=self.user).visible()
return context
def get_value(self):
reuse = (Reuse.objects(metrics__datasets__gt=0).visible().order_by(
'-metrics.datasets').first())
return reuse.metrics['datasets'] if reuse else 0
def get_value(self):
reuse = (Reuse.objects(metrics__followers__gt=0).visible().order_by(
'-metrics.followers').first())
return reuse.metrics['followers'] if reuse else 0
def get_value(self):
return Reuse.objects(organization=self.target).count()
def get_context(self):
context = super(UserReusesView, self).get_context()
context['reuses'] = Reuse.objects(owner=self.user).visible()
return context
def get_value(self):
reuse = (Reuse.objects(metrics__datasets__gt=0).visible()
.order_by('-metrics.datasets').first())
return reuse.metrics['datasets'] if reuse else 0
def get(self):
'''List all my reuses (including private ones)'''
if not current_user.is_authenticated():
api.abort(401)
return list(Reuse.objects(owner=current_user.id))
def get_value(self):
return Reuse.objects(owner=self.user).count()
def get_value(self):
ids = [
d.id for d in Dataset.objects(organization=self.target).only('id')
]
return Reuse.objects(datasets__in=ids).count()
def check_url_does_not_exists(form, field):
'''Ensure a reuse URL is not yet registered'''
if field.data != field.object_data and Reuse.url_exists(field.data):
raise validators.ValidationError(_('This URL is already registered'))
def get_value(self):
ids = [d.id for d in
(Reuse.objects(organization=self.target).only('id') or [])]
return int(Metrics.objects(object_id__in=ids, level='daily')
.sum('values.nb_uniq_visitors'))
def count_reuses(self):
from udata.models import Reuse
self.metrics['reuses'] = Reuse.objects(organization=self).count()
self.save()
def check_url_does_not_exists(form, field):
'''Ensure a reuse URL is not yet registered'''
if field.data != field.object_data and Reuse.url_exists(field.data):
raise validators.ValidationError(_('This URL is already registered'))
def get_value(self):
reuse = Reuse.objects(metrics__followers__gt=0).visible().order_by(
'-metrics.followers').first()
return reuse.metrics.get('followers', 0)
def get_value(self):
return Reuse.objects(organization=self.target).count()
def get_value(self):
reuse = Reuse.objects(metrics__followers__gt=0).visible().order_by('-metrics.followers').first()
return reuse.metrics.get('followers', 0)
def get_value(self):
reuse = (Reuse.objects(metrics__followers__gt=0).visible()
.order_by('-metrics.followers').first())
return reuse.metrics['followers'] if reuse else 0
def get_value(self):
return Reuse.objects(owner=self.user).count()
def get_value(self):
return Reuse.objects(datasets=self.target).visible().count()
def count_reuses(self):
from udata.models import Reuse
self.metrics['reuses'] = Reuse.objects(datasets=self).visible().count()
self.save()
def get_context(self):
context = super(DatasetDetailView, self).get_context()
context['reuses'] = Reuse.objects(datasets=self.dataset)
context['can_edit'] = DatasetEditPermission(self.dataset)
return context
def get_value(self):
ids = [d.id
for d in Dataset.objects(organization=self.target).only('id')]
return Reuse.objects(datasets__in=ids).count()