def sync_ckan_allowed_users_by_resource(*args, **kwargs):
"""Synchroniser `ckan-restricted` pour le cas particuliers
des autorisations par organisation.
"""
def get_all_users_for_organisations(organisation__in):
filter = {'organisation__in': organisation__in,
'organisation__is_active': True}
return [profile.user.username
for profile in Profile.objects.filter(**filter)]
for resource in Resource.objects.exclude(organisations_allowed=None):
dataset = resource.dataset
organisation__in = [r.pk for r in resource.organisations_allowed.all()]
allowed_users = get_all_users_for_organisations(organisation__in)
restricted = {
'allowed_users': ','.join(allowed_users),
'level': 'only_allowed_users'}
ckan_params = {
'id': str(resource.ckan_id),
'restricted': json.dumps(restricted)}
logger.info("Update 'restricted' for Resource '%d'" % resource.pk)
try:
package = CkanHandler.get_package(str(dataset.ckan_id))
CkanHandler.push_resource(package, **ckan_params)
except Exception as e:
logger.exception(e)
logger.info("Continue...")
def clean(self):
# Vérifie la disponibilité du « slug » dans CKAN
slug = self.slug or slugify(self.title)[:100]
ckan_dataset = CkanHandler.get_package(slug)
if ckan_dataset:
if UUID(ckan_dataset['id']
) != self.ckan_id and ckan_dataset['name'] == slug:
raise ValidationError("L'URL du jeu de données est réservé.")
def asynchronous_tasks(filename, dir, content_type, data, user_pk, dataset_pk,
resource_pk, package_id, **kwargs):
if content_type.endswith('zip'):
unzip(filename, dir, flush=True)
else:
raise NotImplementedError('TODO')
username = User.objects.get(pk=user_pk).username
apikey = CkanHandler.get_user(username)['apikey']
html = get_html_content(dataset_pk, resource_pk)
upload = io.BytesIO(html.encode('utf-8'))
data['upload'] = upload
ckan_package = CkanHandler.get_package(package_id)
apikey = CkanHandler.get_user(username)['apikey']
with CkanUserHandler(apikey=apikey) as ckan:
ckan.publish_resource(ckan_package, **data)
upload.close()
def synchronize(instance, with_user=None):
ckan_package = CkanHandler.get_package(str(instance.dataset.ckan_id))
username = with_user and with_user.username or instance.dataset.editor.username
apikey = CkanHandler.get_user(username)['apikey']
location = os.path.join(DIRECTORY_STORAGE, str(instance.pk))
base_url = reverse('resource:directory_storage',
kwargs={
'dataset_id': instance.dataset.pk,
'resource_id': instance.pk
})
files = iterate(location, base_url=base_url)
html = render_to_string('resource/store/ckan_resource_template.html',
context={'files': files})
upload = io.BytesIO(html.encode('utf-8'))
data = {
'id': str(instance.ckan_id),
'url': instance.store.url,
'name': instance.title,
'description': instance.description,
'lang': instance.language,
'data_type': instance.resource_type,
'view_type': 'text_view',
'upload': upload,
'size': '',
'mimetype': 'text/html',
'format': '',
'api': '{}',
'restricted_by_jurisdiction': '',
'extracting_service': 'False',
'crs': '',
'restricted': json.dumps({'level': 'public'}),
}
with CkanUserHandler(apikey=apikey) as ckan:
ckan.publish_resource(ckan_package, **data)
def save_ckan_resource(self, with_user=None):
resource = self.instance
if with_user:
username = with_user.username
else:
username = resource.dataset.editor.username
upload = open(self.filename, 'rb')
data = {
'id': str(resource.ckan_id),
'url': '',
'name': resource.title,
'description': resource.description,
'lang': resource.language,
'data_type': resource.resource_type,
'upload': upload,
'size': resource.upload.file_path.size,
'format': resource.format_type.ckan_format,
'mimetype': resource.format_type.mimetype[0],
'view_type': resource.format_type.ckan_view,
#
'api': '{}',
'restricted_by_jurisdiction': 'False',
'extracting_service': 'False',
'crs': '',
'restricted': json.dumps({'level': 'public'}),
}
ckan_package = CkanHandler.get_package(str(resource.dataset.ckan_id))
apikey = CkanHandler.get_user(username)['apikey']
with CkanUserHandler(apikey=apikey) as ckan:
ckan.publish_resource(ckan_package, **data)
upload.close()
def handle(self, request, *args, **kwargs):
user = request.user
if user.is_anonymous:
profile = None
else:
try:
profile = get_object_or_404(Profile, user=user)
except Exception:
raise ProfileHttp404
qs = request.POST or request.GET
outputformat = qs.get('format')
if not outputformat or outputformat not in ('odl', 'datasud'):
raise Http404()
if outputformat == 'odl':
annotate = OrderedDict((
('COLL_NOM', COLL_NOM),
('COLL_SIRET', COLL_SIRET),
('ID', ID),
('TITRE', TITRE),
('DESCRIPTION', DESCRIPTION),
('THEME', THEME),
# ('DIFFUSEUR', DIFFUSEUR),
('PRODUCTEUR_NOM', PRODUCTEUR_NOM),
('PRODUCTEUR_SIRET', PRODUCTEUR_SIRET),
('COUV_SPAT_MAILLE', COUV_SPAT_MAILLE),
('COUV_SPAT_NOM', COUV_SPAT_NOM),
('COUV_TEMP_DEBUT', COUV_TEMP_DEBUT),
('COUV_TEMP_FIN', COUV_TEMP_DEBUT),
('DATE_PUBL', DATE_PUBL),
('FREQ_MAJ', FREQ_MAJ),
('DATE_MAJ', DATE_MAJ),
('MOTS_CLES', MOTS_CLES),
('LICENCE', LICENCE),
('NOMBRE_RESSOURCES', NOMBRE_RESSOURCES),
('FORMAT_RESSOURCES', FORMAT_RESSOURCES),
# ('PROJECTION', PROJECTION),
# ('LANG', LANG),
('URL', URL)))
else:
annotate = OrderedDict((
('COLL_NOM', COLL_NOM),
('COLL_SIRET', COLL_SIRET),
('ID', ID),
('TITRE', TITRE),
('DESCRIPTION', DESCRIPTION),
('THEME', THEME),
('PRODUCTEUR_NOM', PRODUCTEUR_NOM),
('PRODUCTEUR_SIRET', PRODUCTEUR_SIRET),
('COUV_SPAT_MAILLE', COUV_SPAT_MAILLE),
('COUV_SPAT_NOM', COUV_SPAT_NOM),
('COUV_TEMP_DEBUT', COUV_TEMP_DEBUT),
('COUV_TEMP_FIN', COUV_TEMP_DEBUT),
('DATE_PUBL', DATE_PUBL),
('FREQ_MAJ', FREQ_MAJ),
('DATE_MAJ', DATE_MAJ),
('MOTS_CLES', MOTS_CLES),
('LICENCE', LICENCE),
('NOMBRE_RESSOURCES', NOMBRE_RESSOURCES),
('FORMAT_RESSOURCES', FORMAT_RESSOURCES),
('URL', URL),
('DATASUD_ID', DATASUD_ID),
# ('DATASUD_MOT_CLES', DATASUD_MOT_CLES),
# ('DATASUD_ORGA', DATASUD_ORGA),
('DATASUD_ORGA_ID', DATASUD_ORGA_ID),
('DATASUD_ORGA_URL', DATASUD_ORGA_URL),
('DATASUD_PRODUCTEUR_NOM', DATASUD_PRODUCTEUR_NOM),
# ('DATASUD_PRODUCTEUR_EMAIL', DATASUD_PRODUCTEUR_EMAIL),
('DATASUD_DIFFUSEUR_NOM', DATASUD_DIFFUSEUR_NOM),
# ('DATASUD_DIFFUSEUR_EMAIL', DATASUD_DIFFUSEUR_EMAIL),
('DATASUD_COUV_TERR', DATASUD_COUV_TERR),
# ('DATASUD_INSPIRE', DATASUD_INSPIRE),
# ('DATASUD_DATASET_URL', DATASUD_DATASET_URL),
# ('DATASUD_INSPIRE_URL', DATASUD_INSPIRE_URL),
('DATASUD_DATE_CREATION', DATASUD_DATE_CREATION),
# ('DATASUD_RESSOURCE_URLS', DATASUD_RESSOURCE_URLS),
# ('DATASUD_RESSOURCE_TAILLE', DATASUD_RESSOURCE_TAILLE),
('DATASUD_RESSOURCE_TYPES', DATASUD_RESSOURCE_TYPES),
('DATASUD_DATASET_VUES', DATASUD_DATASET_VUES),
('DATASUD_RESSOURCES_TELECHARGEMENT',
DATASUD_RESSOURCES_TELECHARGEMENT),
('DATASUD_DATASET_NOTE', DATASUD_DATASET_NOTE),
('DATASUD_DATASET_NB_NOTES', DATASUD_DATASET_NB_NOTES),
('DIFFUSEUR', DIFFUSEUR),
('PROJECTION', PROJECTION),
('LANG', LANG),
))
values = list(annotate.keys())
if not profile:
ids = qs.get('ids', '').split(',')
datasets = Dataset.objects.filter(
ckan_id__in=[UUID(id) for id in ids])
elif 'mode' in qs:
mode = qs.get('mode')
if mode == 'all':
roles = profile.get_roles()
if roles['is_admin']:
QuerySet = Dataset.default.all()
elif roles['is_referent']:
kwargs = {
'profile': profile,
'validated_on__isnull': False
}
organisation__in = set(
instance.organisation
for instance in LiaisonsReferents.objects.filter(
**kwargs))
filter = ior(Q(editor=user),
Q(organisation__in=organisation__in))
QuerySet = Dataset.default.filter(filter)
elif mode == 'mine':
QuerySet = Dataset.default.filter(editor=user)
elif mode == 'ckan_harvested':
QuerySet = Dataset.harvested_ckan
elif mode == 'csw_harvested':
QuerySet = Dataset.harvested_csw
else:
raise Http404()
datasets = get_filtered_datasets(QuerySet, qs)
response = HttpResponse(content_type='text/csv')
response[
'Content-Disposition'] = 'attachment; filename=dataset_export.csv'
response['Cache-Control'] = 'no-cache'
writer = unicodecsv.writer(response,
encoding='utf-8',
quoting=csv.QUOTE_ALL,
delimiter=',',
quotechar='"')
writer.writerow(values)
for row in datasets.annotate(**annotate).values(*values):
if not outputformat == 'odl':
package = CkanHandler.get_package(str(row['ID']),
include_tracking=True)
dataset_view = 0
if 'tracking_summary' in package:
dataset_view = package['tracking_summary'].get('total')
row['DATASUD_DATASET_VUES'] = dataset_view
resources_dl = 0
for resource in package.get('resources'):
if 'tracking_summary' in resource:
resources_dl += int(
resource['tracking_summary'].get('total'))
row['DATASUD_RESSOURCES_TELECHARGEMENT'] = resources_dl
row['DATASUD_DATASET_NOTE'] = package.get('rating')
row['DATASUD_DATASET_NB_NOTES'] = package.get('ratings_count')
writer.writerow([row[value] for value in values])
return response
def synchronize(self,
url=None,
filename=None,
content_type=None,
file_extras=None,
with_user=None):
"""Synchronizer le jeu de données avec l'instance de CKAN."""
# Identifiant de la resource CKAN :
id = str(self.ckan_id)
# Définition des propriétés du « package » :
data = {
'crs': self.crs and self.crs.description or '',
'name': self.title,
'description': self.description,
'data_type': self.data_type,
'extracting_service': 'False', # I <3 CKAN
'format': self.format_type and self.format_type.ckan_format,
'view_type': self.format_type and self.format_type.ckan_view,
'id': id,
'lang': self.lang,
'restricted_by_jurisdiction': str(self.geo_restriction),
'url': url and url or '',
'api': '{}'
}
# TODO: Factoriser
# (0) Aucune restriction
if self.restricted_level == 'public':
restricted = json.dumps({'level': 'public'})
# (1) Uniquement pour un utilisateur connecté
elif self.restricted_level == 'registered':
restricted = json.dumps({'level': 'registered'})
# (2) Seulement les utilisateurs indiquées
elif self.restricted_level == 'only_allowed_users':
restricted = json.dumps({
'allowed_users':
','.join(
self.profiles_allowed.exists()
and [p.user.username
for p in self.profiles_allowed.all()] or []),
'level':
'only_allowed_users'
})
# (3) Les utilisateurs de cette organisation
elif self.restricted_level == 'same_organization':
restricted = json.dumps({
'allowed_users':
','.join(
get_all_users_for_organisations(
self.organisations_allowed.all())),
'level':
'only_allowed_users'
})
# (3) Les utilisateurs des organisations indiquées
elif self.restricted_level == 'any_organization':
restricted = json.dumps({
'allowed_users':
','.join(
get_all_users_for_organisations(
self.organisations_allowed.all())),
'level':
'only_allowed_users'
})
data['restricted'] = restricted
if self.referenced_url:
data['url'] = self.referenced_url
if self.dl_url and filename:
downloaded_file = File(open(filename, 'rb'))
data['upload'] = downloaded_file
data['size'] = downloaded_file.size
data['mimetype'] = content_type
if self.up_file and file_extras:
data['upload'] = self.up_file.file
data['size'] = file_extras.get('size')
data['mimetype'] = file_extras.get('mimetype')
if self.ftp_file:
if not url:
data['upload'] = self.ftp_file.file
data['size'] = self.ftp_file.size
data['mimetype'] = None # TODO
if self.data_type == 'raw':
if self.ftp_file or self.dl_url or self.up_file:
data['resource_type'] = 'file.upload'
elif self.referenced_url:
data['resource_type'] = 'file'
if self.data_type == 'annexe':
data['resource_type'] = 'documentation'
if self.data_type == 'service':
data['resource_type'] = 'api'
ckan_package = CkanHandler.get_package(str(self.dataset.ckan_id))
if with_user:
username = with_user.username
apikey = CkanHandler.get_user(username)['apikey']
with CkanUserHandler(apikey=apikey) as ckan:
ckan.publish_resource(ckan_package, **data)
else:
return CkanHandler.publish_resource(ckan_package, **data)
