def delete(self, *args, current_user=None, **kwargs):
with_user = current_user
# On supprime toutes les ressources attachées au jeu de données
Resource = apps.get_model(app_label='idgo_admin',
model_name='Resource')
for resource in Resource.objects.filter(dataset=self):
resource.delete(current_user=current_user,
synchronize_dataset=False)
# > > > > > > BETA < < < < < < #
if BETA:
ResourceBeta = apps.get_model(app_label='idgo_resource',
model_name='Resource')
for resource_beta in ResourceBeta.objects.filter(dataset=self):
resource_beta._related.delete()
resource_beta.delete()
# > > > > > > BETA < < < < < < #
# On supprime le package CKAN
ckan_id = str(self.ckan_id)
if with_user:
username = with_user.username
apikey = CkanHandler.get_user(username)['apikey']
with CkanUserHandler(apikey=apikey) as ckan_user:
ckan_user.delete_dataset(ckan_id)
else:
CkanHandler.delete_dataset(ckan_id)
CkanHandler.purge_dataset(ckan_id)
# On supprime l'instance
super().delete(*args, **kwargs)
def delete(self, *args, current_user=None, **kwargs):
with_user = current_user
# On supprime la ressource CKAN
if with_user:
username = with_user.username
apikey = CkanHandler.get_user(username)['apikey']
with CkanUserHandler(apikey=apikey) as ckan_user:
ckan_user.delete_resource(self.name)
else:
CkanHandler.delete_resource(self.name)
# On supprime les ressources MRA
try:
MRAHandler.del_layer(self.name)
ws_name = self.resource.dataset.organisation.slug
if self.type == 'vector':
MRAHandler.del_featuretype(ws_name, 'public', self.name)
if self.type == 'raster':
MRAHandler.del_coverage(ws_name, self.name, self.name)
# MRAHandler.del_coveragestore(ws_name, self.name)
except Exception as e:
logger.error(e)
pass
# On supprime la table de données PostGIS
try:
drop_table(self.name)
except Exception as e:
logger.error(e)
pass
# Puis on supprime l'instance
super().delete(*args, **kwargs)
def synchronize(self, with_user=None):
"""Synchronizer le jeu de données avec l'instance de CKAN."""
# 'with_user' n'est pas utiliser dans ce contexte
# Définition des propriétés de la « ressource »
# =============================================
id = self.name
name = 'Aperçu cartographique'.format(name=self.resource.title)
description = self.resource.description
organisation = self.resource.dataset.organisation
base_url = OWS_URL_PATTERN.format(
organisation=organisation.slug).replace('?', '')
getlegendgraphic = (
'{base_url}?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetLegendGraphic'
'&LAYER={layer}&FORMAT=image/png').format(base_url=base_url,
layer=id)
api = {
'url': base_url,
'typename': id,
'getlegendgraphic': getlegendgraphic
}
try:
DEFAULT_SRID = settings.DEFAULTS_VALUES['SRID']
except Exception:
DEFAULT_SRID = 4326
else:
SupportedCrs = apps.get_model(app_label='idgo_admin',
model_name='SupportedCrs')
try:
SupportedCrs.objects.get(auth_name='EPSG',
auth_code=DEFAULT_SRID)
except SupportedCrs.DoesNotExist:
DEFAULT_SRID = 4326
if self.type == 'vector':
if self.resource.format_type.extension.lower() in ('json',
'geojson'):
outputformat = 'shapezip' # Il faudrait être sûr que le format existe avec le même nom !
elif self.resource.format_type.extension.lower() in ('zip', 'tar'):
outputformat = 'geojson' # Il faudrait être sûr que le format existe avec le même nom !
api[outputformat] = (
'{base_url}?SERVICE=WFS&VERSION=2.0.0&REQUEST=GetFeature'
'&TYPENAME={typename}&OUTPUTFORMAT={outputformat}&CRSNAME=EPSG:{srid}'
).format(base_url=base_url,
typename=id,
outputformat=outputformat,
srid=str(DEFAULT_SRID))
CkanHandler.update_resource(str(self.resource.ckan_id),
api=json.dumps(api))
CkanHandler.push_resource_view(title=name,
description=description,
resource_id=str(self.resource.ckan_id),
view_type='geo_view')
def handle(self, *args, **options):
total = Resource.objects.all().count()
count = 0
for resource in Resource.objects.all().order_by('id'):
count += 1
logger.info("[%d/%d] - Check Resource '%d'." %
(count, total, resource.pk))
if not resource.up_file:
continue
# else:
url = urljoin(resource.ckan_url,
'download/%s' % resource.up_file.name)
CkanHandler.update_resource(str(resource.ckan_id), url=url)
try:
CkanHandler.update_resource(str(resource.ckan_id), url=url)
except Exception as e:
logger.error("Error with Resource '%d' - '%s'." %
(resource.pk, resource.title))
logger.exception(e)
logger.warning("Continue...")
logger.info("Updated Resource '%d' - '%s' (filename: '%s')." %
(resource.pk, resource.title, resource.up_file.name))
def sync_ckan_allowed_users_by_resource(*args, **kwargs):
"""Synchroniser `ckan-restricted` pour le cas particuliers
des autorisations par organisation.
"""
def get_all_users_for_organisations(organisation__in):
filter = {'organisation__in': organisation__in,
'organisation__is_active': True}
return [profile.user.username
for profile in Profile.objects.filter(**filter)]
for resource in Resource.objects.exclude(organisations_allowed=None):
dataset = resource.dataset
organisation__in = [r.pk for r in resource.organisations_allowed.all()]
allowed_users = get_all_users_for_organisations(organisation__in)
restricted = {
'allowed_users': ','.join(allowed_users),
'level': 'only_allowed_users'}
ckan_params = {
'id': str(resource.ckan_id),
'restricted': json.dumps(restricted)}
logger.info("Update 'restricted' for Resource '%d'" % resource.pk)
try:
package = CkanHandler.get_package(str(dataset.ckan_id))
CkanHandler.push_resource(package, **ckan_params)
except Exception as e:
logger.exception(e)
logger.info("Continue...")
def delete(self,
*args,
current_user=None,
synchronize_dataset=True,
**kwargs):
with_user = current_user
for layer in self.get_layers():
layer.delete(current_user=current_user)
# On supprime la ressource CKAN
ckan_id = str(self.ckan_id)
if with_user:
username = with_user.username
apikey = CkanHandler.get_user(username)['apikey']
with CkanUserHandler(apikey=apikey) as ckan_user:
ckan_user.delete_resource(ckan_id)
else:
CkanHandler.delete_resource(ckan_id)
# On supprime l'instance
super().delete(*args, **kwargs)
# Ce n'est vraiment pas une bonne idée de synchroniser ici le dataset :
self.dataset.date_modification = timezone.now().date()
self.dataset.save(current_user=current_user,
synchronize=synchronize_dataset,
update_fields=['date_modification'])
def handle_crige_partner(sender, instance, **kwargs):
groupname = 'crige-partner'
username = instance.user.username
if CkanHandler.is_user_exists(username):
if instance.crige_membership:
CkanHandler.add_user_to_partner_group(username, groupname)
else:
CkanHandler.del_user_from_partner_group(username, groupname)
def post_save_organisation(sender, instance, **kwargs):
# Mettre à jour en cascade les profiles (utilisateurs)
Profile = apps.get_model(app_label='idgo_admin', model_name='Profile')
for profile in Profile.objects.filter(organisation=instance):
profile.crige_membership = instance.is_crige_partner
profile.save()
# Synchroniser avec l'organisation CKAN
if CkanHandler.is_organisation_exists(str(instance.ckan_id)):
CkanHandler.update_organisation(instance)
def save(self, commit=True, *args, **kwargs):
user = super().save(commit=False)
try:
CkanHandler.update_user(user)
except Exception as e:
raise forms.ValidationError(
"La modification de l'utilisateur sur CKAN a échoué: {}.".format(e))
if commit:
user.save()
return user
def merge_name(self, request, queryset_tags):
datasets = Dataset.objects.filter(
keywords__in=queryset_tags).distinct()
if 'apply' in request.POST:
form = NewKeywordForm(request.POST)
if form.is_valid():
error = False
name = form.cleaned_data.get('new_name')
tag, created = Tag.objects.get_or_create(name=name)
# WIP
for dataset in datasets:
dataset.keywords.add(tag)
ckan_id = str(dataset.ckan_id)
qs_dataset_keywords = dataset.keywords.all().exclude(
id__in=queryset_tags)
tags = [
*[{
'name': k.name
} for k in qs_dataset_keywords], *[{
'name': tag.name
}]
]
logger.info('Update dataset %d with tags: %s' %
(dataset.pk, tags))
try:
CkanHandler.publish_dataset(id=ckan_id, tags=tags)
except CkanBaseError as e:
logger.exception(e)
error = True
dataset.keywords.remove(tag)
break
else:
continue
if error:
messages.error(request, (
"Une erreur est survenue. "
"Veuillez contacter l'administrateur de la plateforme."
))
else:
queryset_tags.exclude(pk=tag.pk).delete()
messages.info(
request, ("La mise à jour est effectuée avec succès."))
return HttpResponseRedirect(request.get_full_path())
else: # request.GET
form = NewKeywordForm()
# then
template_html = 'admin/idgo_admin/taggit_merge_name.html'
context = {'form': form, 'tags': queryset_tags, 'datasets': datasets}
return render(request, template_html, context=context)
def post(self, request):
try:
user, profile = user_and_profile(request)
except ProfileHttp404:
return HttpResponseRedirect(reverse('server_cas:signIn'))
form = UpdateAccountForm(request.POST, instance=user)
if not form.is_valid():
return render_with_info_profile(request,
self.template,
context={'form': form})
try:
with transaction.atomic():
if form.new_password:
user.set_password(form.new_password)
user.save()
logout(request)
login(request,
user,
backend='django.contrib.auth.backends.ModelBackend')
for field in form.Meta.profile_fields:
setattr(profile, field, form.cleaned_data[field])
profile.save()
for field in form.Meta.user_fields:
setattr(user, field, form.cleaned_data[field])
user.save()
CkanHandler.update_user(user)
except ValidationError as e:
messages.error(request, e.message)
return render_with_info_profile(request,
self.template,
context={'form': form})
except CkanBaseError as e:
form.add_error('__all__', e.__str__())
messages.error(request, e.__str__())
return render_with_info_profile(request,
self.template,
context={'form': form})
messages.success(request, 'Votre compte a bien été mis à jour.')
return render_with_info_profile(request,
self.template,
context={'form': form},
status=200)
def save(self, commit=True, *args, **kwargs):
user = super().save(commit=False)
pass_generated = self.password_generator()
try:
CkanHandler.add_user(user, pass_generated, state='active')
except Exception as e:
raise ValidationError(
"L'ajout de l'utilisateur sur CKAN a échoué: {}.".format(e))
user.set_password(pass_generated)
if commit:
user.save()
return user
def handle(self, *args, **options):
queryset = Category.objects.all().order_by('id')
total = queryset.count()
count = 0
for instance in queryset:
count += 1
logger.info("[%d/%d] - Save Resource %d." %
(count, total, instance.pk))
if CkanHandler.is_group_exists(instance.slug):
CkanHandler.update_group(instance)
logger.info("'%s' is udpated." % instance.slug)
else:
CkanHandler.add_group(instance)
logger.info("'%s' is created." % instance.slug)
def delete_ckan_resource(sender, instance, **kwargs):
"""Supprimer la resource CKAN à la suppression d'une resource."""
apikey = CkanHandler.get_user(instance.dataset.editor.username)['apikey']
with CkanUserHandler(apikey=apikey) as ckan:
ckan.delete_resource(str(instance.ckan_id))
logger.info(
"CKAN Resource \"{pk}\" has been deleted.".format(pk=instance.ckan_id))
def clean(self):
username = self.cleaned_data.get('username')
password = self.cleaned_data.get('password')
if username and password:
try:
self.user = authenticate(request=self.request,
username=username,
password=password)
except Exception as e:
raise ValidationError(e.__str__())
if self.user is None:
self.add_error('username', "Vérifiez votre nom d'utilisateur")
self.add_error('password', 'Vérifiez votre mot de passe')
raise ValidationError('User is not found')
else:
ckan_user = CkanHandler.get_user(username)
if ckan_user and ckan_user['state'] == 'deleted':
self.add_error('username',
"Erreur interne d'authentification")
raise ValidationError('CKAN user is deleted')
if not self.user.is_active:
self.add_error('username', "Ce compte n'est pas activé")
raise ValidationError('User is not activate')
return self.cleaned_data
def clean(self):
if not self.unlock_terms and not self.cleaned_data.get(
'terms_and_conditions'):
self.add_error(
'terms_and_conditions',
"Vous devez accepter les conditions générales d'utilisation.")
username = self.cleaned_data.get('username')
if User.objects.filter(username=username).exists(
) or CkanHandler.is_user_exists(username):
self.add_error('username', "Ce nom d'utilisateur est reservé.")
email = self.cleaned_data.get('email')
if User.objects.filter(email=email).exists():
self.add_error('email', 'Ce courriel est reservé.')
password1 = self.cleaned_data.get('password1')
password2 = self.cleaned_data.get('password2')
if password1 and password2 and password1 != password2:
self.add_error('password2', 'Vérifiez les mots de passes.')
self.cleaned_data['password'] = self.cleaned_data.pop('password1')
if IDGO_CONTRIBUTION_REDUCED_TO_PARTNERS:
organisation = self.cleaned_data['organisation']
if organisation and not organisation.is_crige_partner:
self.cleaned_data['contributor'] = False
self.cleaned_data['referent'] = False
return self.cleaned_data
def clean(self):
# Vérifie la disponibilité du « slug » dans CKAN
slug = self.slug or slugify(self.title)[:100]
ckan_dataset = CkanHandler.get_package(slug)
if ckan_dataset:
if UUID(ckan_dataset['id']
) != self.ckan_id and ckan_dataset['name'] == slug:
raise ValidationError("L'URL du jeu de données est réservé.")
def asynchronous_tasks(filename, dir, content_type, data, user_pk, dataset_pk,
resource_pk, package_id, **kwargs):
if content_type.endswith('zip'):
unzip(filename, dir, flush=True)
else:
raise NotImplementedError('TODO')
username = User.objects.get(pk=user_pk).username
apikey = CkanHandler.get_user(username)['apikey']
html = get_html_content(dataset_pk, resource_pk)
upload = io.BytesIO(html.encode('utf-8'))
data['upload'] = upload
ckan_package = CkanHandler.get_package(package_id)
apikey = CkanHandler.get_user(username)['apikey']
with CkanUserHandler(apikey=apikey) as ckan:
ckan.publish_resource(ckan_package, **data)
upload.close()
def synchronize(instance, with_user=None):
ckan_package = CkanHandler.get_package(str(instance.dataset.ckan_id))
username = with_user and with_user.username or instance.dataset.editor.username
apikey = CkanHandler.get_user(username)['apikey']
location = os.path.join(DIRECTORY_STORAGE, str(instance.pk))
base_url = reverse('resource:directory_storage',
kwargs={
'dataset_id': instance.dataset.pk,
'resource_id': instance.pk
})
files = iterate(location, base_url=base_url)
html = render_to_string('resource/store/ckan_resource_template.html',
context={'files': files})
upload = io.BytesIO(html.encode('utf-8'))
data = {
'id': str(instance.ckan_id),
'url': instance.store.url,
'name': instance.title,
'description': instance.description,
'lang': instance.language,
'data_type': instance.resource_type,
'view_type': 'text_view',
'upload': upload,
'size': '',
'mimetype': 'text/html',
'format': '',
'api': '{}',
'restricted_by_jurisdiction': '',
'extracting_service': 'False',
'crs': '',
'restricted': json.dumps({'level': 'public'}),
}
with CkanUserHandler(apikey=apikey) as ckan:
ckan.publish_resource(ckan_package, **data)
def handle(self, *args, **options):
dataset_qs = Dataset.default.filter(
keywords__isnull=False).distinct().order_by('id')
total = dataset_qs.count()
count = 0
for dataset in dataset_qs:
count += 1
qs_dataset_keywords = dataset.keywords.all()
ckan_id = str(dataset.ckan_id)
logger.info(
"[%d/%d] - Synchronize Dataset %d (%s) with tags: '%s'." %
(count, total, dataset.pk, ckan_id, "', '".join(
[k.name for k in qs_dataset_keywords])))
try:
CkanHandler.publish_dataset(id=ckan_id,
tags=[{
'name': k.name
} for k in qs_dataset_keywords])
except Exception as e:
logger.exception(e)
logger.warning("Error was ingored.")
def save_ckan_resource(self, with_user=None):
resource = self.instance
if with_user:
username = with_user.username
else:
username = resource.dataset.editor.username
upload = open(self.filename, 'rb')
data = {
'id': str(resource.ckan_id),
'url': '',
'name': resource.title,
'description': resource.description,
'lang': resource.language,
'data_type': resource.resource_type,
'upload': upload,
'size': resource.upload.file_path.size,
'format': resource.format_type.ckan_format,
'mimetype': resource.format_type.mimetype[0],
'view_type': resource.format_type.ckan_view,
#
'api': '{}',
'restricted_by_jurisdiction': 'False',
'extracting_service': 'False',
'crs': '',
'restricted': json.dumps({'level': 'public'}),
}
ckan_package = CkanHandler.get_package(str(resource.dataset.ckan_id))
apikey = CkanHandler.get_user(username)['apikey']
with CkanUserHandler(apikey=apikey) as ckan:
ckan.publish_resource(ckan_package, **data)
upload.close()
def clean(self):
if not self.unlock_terms and not self.cleaned_data.get('terms_and_conditions'):
self.add_error('terms_and_conditions', "Vous devez accepter les conditions générales d'utilisation.")
username = self.cleaned_data.get('username')
if User.objects.filter(username=username).exists() or CkanHandler.is_user_exists(username):
self.add_error('username', "Ce nom d'utilisateur est reservé.")
email = self.cleaned_data.get('email')
if User.objects.filter(email=email).exists():
self.add_error('email', 'Ce courriel est reservé.')
password1 = self.cleaned_data.get('password1')
password2 = self.cleaned_data.get('password2')
if password1 and password2 and password1 != password2:
self.add_error('password2', 'Vérifiez les mots de passes.')
self.cleaned_data['password'] = self.cleaned_data.pop('password1')
return self.cleaned_data
def handle(self, *args, **options):
for resource in Resource.objects.exclude(organisations_allowed=None):
dataset = resource.dataset
ckan_params = {
'id':
str(resource.ckan_id),
'restricted':
json.dumps({
'allowed_users':
','.join(
get_all_users_for_organisations([
r.pk for r in resource.organisations_allowed.all()
])),
'level':
'only_allowed_users'
})
}
apikey = CkanHandler.get_user(dataset.editor.username)['apikey']
with CkanUserHandler(apikey=apikey) as ckan:
package = ckan.get_package(str(dataset.ckan_id))
ckan.push_resource(package, **ckan_params)
def get_context(self, form, user, dataset):
layer_rows = []
resource_rows = []
if dataset:
for resource in Resource.objects.filter(dataset=dataset):
resource_row_data = (
resource.pk,
resource.title,
resource.format_type.description if resource.format_type else None,
resource.get_data_type_display(),
resource.created_on.isoformat() if resource.created_on else None,
resource.last_update.isoformat() if resource.last_update else None,
resource.get_restricted_level_display(),
str(resource.ckan_id),
[layer.pk for layer in resource.get_layers()],
resource.ogc_services,
resource.extractable,
)
resource_rows.append(resource_row_data)
common = [
resource.pk,
resource.title,
resource.get_data_type_display(),
resource.get_restricted_level_display(),
resource.geo_restriction,
resource.extractable,
resource.ogc_services,
]
layers = resource.get_layers()
if layers:
for layer in resource.get_layers():
layer_row_data = common.copy()
layer_row_data.extend((
layer.pk,
layer.mra_info['name'],
layer.mra_info['title'],
layer.mra_info['type'],
layer.mra_info['enabled'],
layer.mra_info['bbox'],
layer.mra_info['attributes'],
layer.mra_info['styles'],
))
layer_rows.append(layer_row_data)
licenses = [
(m.pk, m.license.pk) for m
in LiaisonsContributeurs.get_contribs(profile=user.profile) if m.license]
supports = [
(m.pk, {'name': m.name, 'email': m.email})
for m in Support.objects.all()]
tags = CkanHandler.get_tags()
return {
'target': target(dataset, user),
'dataset': dataset,
'form': form,
'licenses': dict(licenses),
'layer_rows': json.dumps(layer_rows),
'resource_rows': json.dumps(resource_rows),
'supports': json.dumps(dict(supports)),
'tags': json.dumps(tags),
}
def save(self, *args, harvest=True, **kwargs):
Category = apps.get_model(app_label='idgo_admin',
model_name='Category')
Dataset = apps.get_model(app_label='idgo_admin', model_name='Dataset')
License = apps.get_model(app_label='idgo_admin', model_name='License')
Resource = apps.get_model(app_label='idgo_admin',
model_name='Resource')
ResourceFormats = apps.get_model(app_label='idgo_admin',
model_name='ResourceFormats')
# (1) Supprimer les jeux de données qui ne sont plus synchronisés
previous = self.pk and RemoteCsw.objects.get(pk=self.pk)
if previous:
for dataset in Dataset.harvested_csw.filter(
remote_instance=previous):
dataset.delete()
else:
# Dans le cas d'une création, on vérifie si l'URL CSW est valide
try:
with CswBaseHandler(self.url):
pass
except CswBaseError as e:
raise ValidationError(e.__str__(), code='url')
# (2) Sauver l'instance
super().save(*args, **kwargs)
# (3) Créer/Mettre à jour les jeux de données synchronisés
# On récupère dans le `stack` l'utilisateur effectuant l'opération
editor = None
for entry in inspect.stack():
try:
editor = entry[0].f_locals['request'].user._wrapped
except (KeyError, AttributeError):
continue
break
if not previous:
return
if harvest:
# Puis on moissonne le catalogue
try:
ckan_ids = []
geonet_ids = []
with transaction.atomic():
with CswBaseHandler(self.url) as csw:
packages = csw.get_packages(
xml=self.getrecords or None)
for package in packages:
if not package['type'] == 'dataset':
continue
geonet_id = package['id']
update_frequency = dict(Dataset.FREQUENCY_CHOICES).get(
package.get('frequency'), 'unknown')
update_frequency = package.get('frequency')
if not (update_frequency and update_frequency in dict(
Dataset.FREQUENCY_CHOICES).keys()):
update_frequency = 'unknown'
date_creation = package.get('dataset_creation_date',
None)
if date_creation:
try:
date_creation = datetime.strptime(
date_creation, ISOFORMAT_DATE)
except ValueError as e:
logger.warning(e)
date_creation = None
date_modification = package.get(
'dataset_modification_date', None)
if date_modification:
try:
date_modification = datetime.strptime(
date_modification, ISOFORMAT_DATE)
except ValueError as e:
logger.warning(e)
date_modification = None
date_publication = package.get(
'dataset_publication_date', None)
if date_publication:
try:
date_publication = datetime.strptime(
date_publication, ISOFORMAT_DATE)
except ValueError as e:
logger.warning(e)
date_publication = None
# Licence
license_titles = package.get('license_titles')
filters = [
Q(slug__in=license_titles),
Q(title__in=license_titles),
Q(alternate_titles__overlap=license_titles),
]
license = License.objects.filter(reduce(
ior, filters)).distinct().first()
if not license:
try:
license = License.objects.get(
slug=settings.DEFAULTS_VALUES.get(
'LICENSE'))
except License.DoesNotExist:
license = License.objects.first()
# On pousse la fiche de MD dans Geonet
if not geonet.get_record(geonet_id):
try:
geonet.create_record(geonet_id, package['xml'])
except Exception as e:
logger.warning(
'La création de la fiche de métadonnées a échoué.'
)
logger.error(e)
else:
geonet_ids.append(geonet_id)
geonet.publish(
geonet_id) # Toujours publier la fiche
else:
try:
geonet.update_record(geonet_id, package['xml'])
except Exception as e:
logger.warning(
'La mise à jour de la fiche de métadonnées a échoué.'
)
logger.error(e)
slug = 'sync{}-{}'.format(
str(uuid.uuid4())[:7].lower(),
slugify(geonet_id))[:100]
kvp = {
'slug':
slug,
'title':
package.get('title'),
'description':
package.get('notes'),
'date_creation':
date_creation and date_creation.date(),
'date_modification':
date_modification and date_modification.date(),
'date_publication':
date_publication and date_publication.date(),
'editor':
editor,
'license':
license,
'owner_email':
self.organisation.email or DEFAULT_CONTACT_EMAIL,
'owner_name':
self.organisation.legal_name
or DEFAULT_PLATFORM_NAME,
'organisation':
self.organisation,
'published':
not package.get('private'),
'remote_instance':
self,
'remote_dataset':
geonet_id,
'update_frequency':
update_frequency,
'bbox':
package.get('bbox'),
# broadcaster_email
# broadcaster_name
# data_type
# geocover
'geonet_id':
geonet_id,
# granularity
# thumbnail
# support
}
dataset, created = Dataset.harvested_csw.update_or_create(
**kvp)
if created:
ckan_ids.append(dataset.ckan_id)
categories_name = [
m['name'] for m in package.get('groups', [])
]
iso_topic_reverse = dict(
(v, k)
for k, v in Category._meta.fields[5].choices)
filters = [
Q(slug__in=categories_name),
Q(name__in=categories_name),
Q(iso_topic__in=[
m['name'] for m in package.get('groups', [])
]),
Q(iso_topic__in=[
iso_topic_reverse.get(name)
for name in categories_name
]),
Q(alternate_titles__overlap=categories_name),
]
categories = Category.objects.filter(
reduce(ior, filters)).distinct()
if categories:
dataset.categories.set(categories, clear=True)
if not created:
dataset.keywords.clear()
keywords = [
tag['display_name'] for tag in package.get('tags')
]
dataset.keywords.add(*keywords)
dataset.save(current_user=None,
synchronize=True,
activate=False)
for resource in package.get('resources', []):
try:
ckan_id = uuid.uuid4()
except ValueError as e:
logger.exception(e)
logger.error(
"I can't crash here, so I do not pay any attention to this error."
)
continue
filters = []
protocol = resource.get('protocol')
protocol and filters.append(Q(protocol=protocol))
mimetype = resource.get('mimetype')
mimetype and filters.append(
Q(mimetype__overlap=[mimetype]))
try:
format_type = ResourceFormats.objects.get(
reduce(iand, filters))
except (ResourceFormats.MultipleObjectsReturned,
ResourceFormats.DoesNotExist, TypeError):
format_type = None
kvp = {
'ckan_id': ckan_id,
'dataset': dataset,
'format_type': format_type,
'title': resource['name'] or resource['url'],
'referenced_url': resource['url'],
}
try:
resource = Resource.objects.get(
ckan_id=ckan_id)
except Resource.DoesNotExist:
resource = Resource.default.create(save_opts={
'current_user':
editor,
'synchronize':
True
},
**kvp)
else:
for k, v in kvp.items():
setattr(resource, k, v)
resource.save(current_user=editor,
synchronize=True)
except Exception as e:
for id in ckan_ids:
logger.warning(
'Delete CKAN package : {id}.'.format(id=str(id)))
CkanHandler.purge_dataset(str(id))
for id in geonet_ids:
logger.warning('Delete MD : {id}.'.format(id=str(id)))
geonet.delete_record(id)
logger.error(e)
raise CriticalError()
else:
for id in ckan_ids:
CkanHandler.publish_dataset(id=str(id), state='active')
def save(self, *args, **kwargs):
Category = apps.get_model(app_label='idgo_admin',
model_name='Category')
Dataset = apps.get_model(app_label='idgo_admin', model_name='Dataset')
License = apps.get_model(app_label='idgo_admin', model_name='License')
Resource = apps.get_model(app_label='idgo_admin',
model_name='Resource')
ResourceFormats = apps.get_model(app_label='idgo_admin',
model_name='ResourceFormats')
# (1) Supprimer les jeux de données qui ne sont plus synchronisés
previous = self.pk and RemoteCkan.objects.get(pk=self.pk)
if previous:
remote_organisation__in = [
x for x in (previous.sync_with or [])
if x not in (self.sync_with or [])
]
filter = {
'remote_instance': previous,
'remote_organisation__in': remote_organisation__in,
}
# TODO: 'Dataset.harvested_ckan.filter(**filter).delete()' ne fonctionne pas
for dataset in Dataset.harvested_ckan.filter(**filter):
dataset.delete()
else:
# Dans le cas d'une création, on vérifie si l'URL CKAN est valide
try:
with CkanBaseHandler(self.url):
pass
except CkanBaseError as e:
raise ValidationError(e.__str__(), code='url')
# (2) Sauver l'instance
super().save(*args, **kwargs)
# (3) Créer/Mettre à jour les jeux de données synchronisés
# On récupère dans le `stack` l'utilisateur effectuant l'opération
editor = User.objects.get(pk=DEFAULT_USER_ID)
for entry in inspect.stack():
try:
editor = entry[0].f_locals['request'].user._wrapped
except (KeyError, AttributeError):
continue
break
# Puis on moissonne le catalogue
if self.sync_with:
try:
ckan_ids = []
with transaction.atomic():
# TODO: Factoriser
for value in self.sync_with:
with CkanBaseHandler(self.url) as ckan:
ckan_organisation = ckan.get_organisation(
value,
include_datasets=True,
include_groups=True,
include_tags=True)
if not ckan_organisation.get('package_count', 0):
continue
for package in ckan_organisation.get('packages'):
if not package['state'] == 'active' \
or not package['type'] == 'dataset':
continue
with CkanBaseHandler(self.url) as ckan:
package = ckan.get_package(package['id'])
ckan_id = uuid.UUID(package['id'])
update_frequency = dict(
Dataset.FREQUENCY_CHOICES).get(
package.get('frequency'), 'unknown')
update_frequency = package.get('frequency')
if not (update_frequency and update_frequency
in dict(Dataset.FREQUENCY_CHOICES).keys()):
update_frequency = 'unknown'
metadata_created = package.get(
'metadata_created', None)
if metadata_created:
metadata_created = datetime.strptime(
metadata_created, ISOFORMAT_DATETIME)
metadata_modified = package.get(
'metadata_modified', None)
if metadata_modified:
metadata_modified = datetime.strptime(
metadata_modified, ISOFORMAT_DATETIME)
try:
mapping_licence = MappingLicence.objects.get(
remote_ckan=self,
slug=package.get('license_id'))
except MappingLicence.DoesNotExist:
try:
license = License.objects.get(
slug='other-at')
except MappingLicence.DoesNotExist:
license = None
else:
logger.warning("'{}' non trouvé".format(
package.get('license_id')))
license = mapping_licence.licence
slug = 'sync{}-{}'.format(
str(uuid.uuid4())[:7].lower(),
package.get('name'))[:100]
kvp = {
'slug':
slug,
'title':
package.get('title'),
'description':
package.get('notes'),
'date_creation':
metadata_created and metadata_created.date(),
'date_modification':
metadata_modified and metadata_modified.date(),
# date_publication
'editor':
editor,
'license':
license,
'owner_email':
self.organisation.email
or DEFAULT_CONTACT_EMAIL,
'owner_name':
self.organisation.legal_name
or DEFAULT_PLATFORM_NAME,
'organisation':
self.organisation,
'published':
not package.get('private'),
'remote_instance':
self,
'remote_dataset':
ckan_id,
'remote_organisation':
value,
'update_frequency':
update_frequency,
# bbox
# broadcaster_email
# broadcaster_name
# data_type
# geocover
# geonet_id
# granularity
# thumbnail
# support
}
dataset, created = Dataset.harvested_ckan.update_or_create(
**kvp)
mapping_categories = MappingCategory.objects.filter(
remote_ckan=self,
slug__in=[
m['name']
for m in package.get('groups', [])
])
if mapping_categories:
dataset.categories = set(
mc.category for mc in mapping_categories)
if not created:
dataset.keywords.clear()
keywords = [
tag['display_name']
for tag in package.get('tags')
]
dataset.keywords.add(*keywords)
dataset.save(current_user=None,
synchronize=True,
activate=False)
ckan_ids.append(dataset.ckan_id)
for resource in package.get('resources', []):
try:
ckan_id = uuid.UUID(resource['id'])
except ValueError as e:
logger.exception(e)
logger.error(
"I can't crash here, so I do not pay any attention to this error."
)
continue
try:
ckan_format = resource['format'].upper()
format_type = ResourceFormats.objects.get(
ckan_format=ckan_format)
except (ResourceFormats.
MultipleObjectsReturned,
ResourceFormats.DoesNotExist,
TypeError) as e:
logger.exception(e)
logger.error(
"I can't crash here, so I do not pay any attention to this error."
)
format_type = None
kvp = {
'ckan_id': ckan_id,
'dataset': dataset,
'format_type': format_type,
'title': resource['name'],
'referenced_url': resource['url'],
}
try:
resource = Resource.objects.get(
ckan_id=ckan_id)
except Resource.DoesNotExist:
resource = Resource.default.create(
save_opts={
'current_user': None,
'synchronize': True
},
**kvp)
else:
for k, v in kvp.items():
setattr(resource, k, v)
resource.save(current_user=None,
synchronize=True)
except Exception as e:
for id in ckan_ids:
CkanHandler.purge_dataset(str(id))
logger.error(e)
raise CriticalError()
else:
for id in ckan_ids:
CkanHandler.publish_dataset(id=str(id), state='active')
def post_delete_organisation(sender, instance, **kwargs):
if CkanHandler.is_organisation_exists(str(instance.ckan_id)):
CkanHandler.purge_organisation(str(instance.ckan_id))
def post(self, request, *args, **kwargs):
user, profile = user_and_profile(request)
if not profile.crige_membership:
raise Http404
context = self.get_context(request, user)
footprint = request.POST.get('footprint') or None
footprint = footprint and json.loads(footprint)
layer_name = request.POST.get('layer')
resource_name = request.POST.get('resource')
dataset_name = request.POST.get('dataset')
dst_crs = request.POST.get('crs')
format_vector = request.POST.get('format-vector') or None
if format_vector:
dst_format_vector = ExtractorSupportedFormat.objects.get(
name=format_vector, type='vector').details
format_raster = request.POST.get('format-raster') or None
if format_raster:
dst_format_raster = ExtractorSupportedFormat.objects.get(
name=format_raster, type='raster').details
data_extractions = []
additional_files = []
if layer_name or resource_name:
if layer_name:
model = 'Layer'
foreign_field = 'name'
foreign_value = layer_name
layer = get_object_or_404(Layer, name=layer_name)
resource = layer.resource
if resource_name:
model = 'Resource'
foreign_field = 'ckan_id'
foreign_value = resource_name
resource = get_object_or_404(Resource, ckan_id=resource_name)
layer = resource.get_layers()[0] # Car relation 1-1
if layer.type == 'raster':
data_extraction = {
**{
'source': layer.filename,
},
**dst_format_raster
}
elif layer.type == 'vector':
data_extraction = {
**{
'layer':
layer.name,
'source':
'PG:host=postgis-master user=datagis dbname=datagis',
},
**dst_format_vector
}
data_extraction['dst_srs'] = dst_crs or 'EPSG:2154'
if resource.geo_restriction:
footprint_restriction = \
json.loads(user.profile.organisation.jurisdiction.geom.geojson)
if footprint:
try:
data_extraction['footprint'] = intersect(
json.dumps(footprint),
json.dumps(footprint_restriction))
except Exception as e:
msg = "La zone d'extraction génère une erreur"
messages.error(request, msg)
return render_with_info_profile(request,
self.template,
context=context)
else:
data_extraction['footprint'] = footprint_restriction
data_extraction['footprint_srs'] = 'EPSG:4326'
elif footprint:
data_extraction['footprint'] = footprint
data_extraction['footprint_srs'] = 'EPSG:4326'
data_extractions.append(data_extraction)
# Pas d'`additional_files` dans le cas présent.
elif dataset_name:
model = 'Dataset'
foreign_field = 'slug'
foreign_value = dataset_name
dataset = get_object_or_404(Dataset, slug=dataset_name)
for resource in dataset.get_resources():
for layer in resource.get_layers():
if layer.type == 'raster':
data_extraction = {
**{
'source': layer.filename,
},
**dst_format_raster
}
elif layer.type == 'vector':
data_extraction = {
**{
'layer':
layer.name,
'source':
'PG:host=postgis-master user=datagis dbname=datagis',
},
**dst_format_vector
}
data_extraction['dst_srs'] = dst_crs or 'EPSG:2154'
if resource.geo_restriction:
footprint_restriction = \
json.loads(user.profile.organisation.jurisdiction.geom.geojson)
if footprint:
data_extraction['footprint'] = intersect(
json.dumps(footprint),
json.dumps(footprint_restriction))
else:
data_extraction[
'footprint'] = footprint_restriction
data_extraction['footprint_srs'] = 'EPSG:4326'
elif footprint:
data_extraction['footprint'] = footprint
data_extraction['footprint_srs'] = 'EPSG:4326'
data_extractions.append(data_extraction)
if resource.data_type == 'annexe':
additional_files.append({
'file_name':
resource.filename,
'dir_name':
'Documentation associée',
'file_location':
CkanHandler.get_resource(str(
resource.ckan_id)).get('url')
})
query = {
'user_id':
user.username,
'user_email_address':
user.email,
'user_name':
user.last_name,
'user_first_name':
user.first_name,
'user_company':
user.profile.organisation and user.profile.organisation.legal_name
or '',
'user_address':
user.profile.organisation
and user.profile.organisation.full_address or '',
'data_extractions':
data_extractions,
'additional_files':
additional_files
}
r = requests.post(EXTRACTOR_URL, json=query)
if r.status_code == 201:
details = r.json()
AsyncExtractorTask.objects.create(
details=details,
foreign_field=foreign_field,
foreign_value=foreign_value,
model=model,
query=query,
submission_datetime=details.get('submission_datetime'),
uuid=UUID(details.get('task_id')),
user=user)
messages.success(request, (
"L'extraction a été ajoutée à la liste de tâche. "
"Vous allez recevoir un e-mail une fois l'extraction réalisée."
))
domain = Site.objects.get(name='extractor').domain
url = 'http{secure}://{domain}{path}'.format(
secure=request.is_secure and 's' or '',
domain=domain,
path=reverse('idgo_admin:extractor_dashboard'))
return HttpResponseRedirect(url)
else:
if r.status_code == 400:
details = r.json().get('detail')
msg = '{}: {}'.format(details.get('title', 'Error'),
' '.join(details.get('list', 'Error')))
else:
msg = "L'extracteur n'est pas disponible pour le moment."
messages.error(request, msg)
return render_with_info_profile(request,
self.template,
context=context)
def confirmation_mail(request, key):
action = get_object_or_404(AccountActions,
key=UUID(key),
action='confirm_mail')
if action.closed:
message = 'Vous avez déjà validé votre adresse e-mail.'
return render(request,
'idgo_admin/message.html', {'message': message},
status=200)
user = action.profile.user
profile = action.profile
organisation = profile.organisation
CkanHandler.activate_user(user.username)
user.is_active = True
action.profile.is_active = True
user.save()
action.profile.save()
if organisation:
# Demande de création d'une nouvelle organisation
if not organisation.is_active:
new_organisation_action = get_object_or_404(
AccountActions,
action='confirm_new_organisation',
organisation=organisation,
profile=profile,
closed=None)
url = request.build_absolute_uri(
reverse('idgo_admin:confirm_new_orga',
kwargs={'key': new_organisation_action.key}))
send_organisation_creation_confirmation_mail(
user, organisation, url)
# Demande de rattachement (Profile-Organisation)
rattachement_action = get_object_or_404(AccountActions,
action='confirm_rattachement',
organisation=organisation,
profile=profile,
closed=None)
url = request.build_absolute_uri(
reverse('idgo_admin:confirm_rattachement',
kwargs={'key': rattachement_action.key}))
send_membership_confirmation_mail(user, organisation, url)
# Demande de rôle de référent en attente de validation
try:
LiaisonsReferents.objects.get(organisation=organisation,
profile=profile,
validated_on=None)
except Exception:
pass
else:
referent_action = get_object_or_404(AccountActions,
action='confirm_referent',
organisation=organisation,
profile=profile,
closed=None)
url = request.build_absolute_uri(
reverse('idgo_admin:confirm_referent',
kwargs={'key': referent_action.key}))
send_referent_confirmation_mail(user, organisation, url)
# Demande de rôle de contributeur en attente de validation
try:
LiaisonsContributeurs.objects.get(profile=profile,
organisation=organisation,
validated_on=None)
except Exception:
pass
else:
contribution_action = get_object_or_404(
AccountActions,
action='confirm_contribution',
organisation=organisation,
profile=profile,
closed=None)
url = request.build_absolute_uri(
reverse('idgo_admin:confirm_contribution',
kwargs={'key': contribution_action.key}))
send_contributor_confirmation_mail(user, organisation, url)
send_successful_account_creation_mail(user)
action.closed = timezone.now()
action.save()
message = ("Merci d'avoir confirmé votre adresse e-mail. "
'Toute demande de rattachement, contribution, '
'ou rôle de référent technique pour une organisation, '
"ne sera effective qu'après validation "
'par un administrateur.')
context = {
'message': message,
'button': {
'href': reverse('server_cas:signIn'),
'label': 'Se connecter'
}
}
return render(request, 'idgo_admin/message.html', context, status=200)
def handle(self, request, *args, **kwargs):
user = request.user
if user.is_anonymous:
profile = None
else:
try:
profile = get_object_or_404(Profile, user=user)
except Exception:
raise ProfileHttp404
qs = request.POST or request.GET
outputformat = qs.get('format')
if not outputformat or outputformat not in ('odl', 'datasud'):
raise Http404()
if outputformat == 'odl':
annotate = OrderedDict((
('COLL_NOM', COLL_NOM),
('COLL_SIRET', COLL_SIRET),
('ID', ID),
('TITRE', TITRE),
('DESCRIPTION', DESCRIPTION),
('THEME', THEME),
# ('DIFFUSEUR', DIFFUSEUR),
('PRODUCTEUR_NOM', PRODUCTEUR_NOM),
('PRODUCTEUR_SIRET', PRODUCTEUR_SIRET),
('COUV_SPAT_MAILLE', COUV_SPAT_MAILLE),
('COUV_SPAT_NOM', COUV_SPAT_NOM),
('COUV_TEMP_DEBUT', COUV_TEMP_DEBUT),
('COUV_TEMP_FIN', COUV_TEMP_DEBUT),
('DATE_PUBL', DATE_PUBL),
('FREQ_MAJ', FREQ_MAJ),
('DATE_MAJ', DATE_MAJ),
('MOTS_CLES', MOTS_CLES),
('LICENCE', LICENCE),
('NOMBRE_RESSOURCES', NOMBRE_RESSOURCES),
('FORMAT_RESSOURCES', FORMAT_RESSOURCES),
# ('PROJECTION', PROJECTION),
# ('LANG', LANG),
('URL', URL)))
else:
annotate = OrderedDict((
('COLL_NOM', COLL_NOM),
('COLL_SIRET', COLL_SIRET),
('ID', ID),
('TITRE', TITRE),
('DESCRIPTION', DESCRIPTION),
('THEME', THEME),
('PRODUCTEUR_NOM', PRODUCTEUR_NOM),
('PRODUCTEUR_SIRET', PRODUCTEUR_SIRET),
('COUV_SPAT_MAILLE', COUV_SPAT_MAILLE),
('COUV_SPAT_NOM', COUV_SPAT_NOM),
('COUV_TEMP_DEBUT', COUV_TEMP_DEBUT),
('COUV_TEMP_FIN', COUV_TEMP_DEBUT),
('DATE_PUBL', DATE_PUBL),
('FREQ_MAJ', FREQ_MAJ),
('DATE_MAJ', DATE_MAJ),
('MOTS_CLES', MOTS_CLES),
('LICENCE', LICENCE),
('NOMBRE_RESSOURCES', NOMBRE_RESSOURCES),
('FORMAT_RESSOURCES', FORMAT_RESSOURCES),
('URL', URL),
('DATASUD_ID', DATASUD_ID),
# ('DATASUD_MOT_CLES', DATASUD_MOT_CLES),
# ('DATASUD_ORGA', DATASUD_ORGA),
('DATASUD_ORGA_ID', DATASUD_ORGA_ID),
('DATASUD_ORGA_URL', DATASUD_ORGA_URL),
('DATASUD_PRODUCTEUR_NOM', DATASUD_PRODUCTEUR_NOM),
# ('DATASUD_PRODUCTEUR_EMAIL', DATASUD_PRODUCTEUR_EMAIL),
('DATASUD_DIFFUSEUR_NOM', DATASUD_DIFFUSEUR_NOM),
# ('DATASUD_DIFFUSEUR_EMAIL', DATASUD_DIFFUSEUR_EMAIL),
('DATASUD_COUV_TERR', DATASUD_COUV_TERR),
# ('DATASUD_INSPIRE', DATASUD_INSPIRE),
# ('DATASUD_DATASET_URL', DATASUD_DATASET_URL),
# ('DATASUD_INSPIRE_URL', DATASUD_INSPIRE_URL),
('DATASUD_DATE_CREATION', DATASUD_DATE_CREATION),
# ('DATASUD_RESSOURCE_URLS', DATASUD_RESSOURCE_URLS),
# ('DATASUD_RESSOURCE_TAILLE', DATASUD_RESSOURCE_TAILLE),
('DATASUD_RESSOURCE_TYPES', DATASUD_RESSOURCE_TYPES),
('DATASUD_DATASET_VUES', DATASUD_DATASET_VUES),
('DATASUD_RESSOURCES_TELECHARGEMENT',
DATASUD_RESSOURCES_TELECHARGEMENT),
('DATASUD_DATASET_NOTE', DATASUD_DATASET_NOTE),
('DATASUD_DATASET_NB_NOTES', DATASUD_DATASET_NB_NOTES),
('DIFFUSEUR', DIFFUSEUR),
('PROJECTION', PROJECTION),
('LANG', LANG),
))
values = list(annotate.keys())
if not profile:
ids = qs.get('ids', '').split(',')
datasets = Dataset.objects.filter(
ckan_id__in=[UUID(id) for id in ids])
elif 'mode' in qs:
mode = qs.get('mode')
if mode == 'all':
roles = profile.get_roles()
if roles['is_admin']:
QuerySet = Dataset.default.all()
elif roles['is_referent']:
kwargs = {
'profile': profile,
'validated_on__isnull': False
}
organisation__in = set(
instance.organisation
for instance in LiaisonsReferents.objects.filter(
**kwargs))
filter = ior(Q(editor=user),
Q(organisation__in=organisation__in))
QuerySet = Dataset.default.filter(filter)
elif mode == 'mine':
QuerySet = Dataset.default.filter(editor=user)
elif mode == 'ckan_harvested':
QuerySet = Dataset.harvested_ckan
elif mode == 'csw_harvested':
QuerySet = Dataset.harvested_csw
else:
raise Http404()
datasets = get_filtered_datasets(QuerySet, qs)
response = HttpResponse(content_type='text/csv')
response[
'Content-Disposition'] = 'attachment; filename=dataset_export.csv'
response['Cache-Control'] = 'no-cache'
writer = unicodecsv.writer(response,
encoding='utf-8',
quoting=csv.QUOTE_ALL,
delimiter=',',
quotechar='"')
writer.writerow(values)
for row in datasets.annotate(**annotate).values(*values):
if not outputformat == 'odl':
package = CkanHandler.get_package(str(row['ID']),
include_tracking=True)
dataset_view = 0
if 'tracking_summary' in package:
dataset_view = package['tracking_summary'].get('total')
row['DATASUD_DATASET_VUES'] = dataset_view
resources_dl = 0
for resource in package.get('resources'):
if 'tracking_summary' in resource:
resources_dl += int(
resource['tracking_summary'].get('total'))
row['DATASUD_RESSOURCES_TELECHARGEMENT'] = resources_dl
row['DATASUD_DATASET_NOTE'] = package.get('rating')
row['DATASUD_DATASET_NB_NOTES'] = package.get('ratings_count')
writer.writerow([row[value] for value in values])
return response
