def post(self, request, id, *args, **kwargs):
instance = get_object_or_404(Organisation, id=id, is_active=True)
if not request.is_ajax():
if 'delete' in request.POST and instance.geonet_id:
try:
geonet.delete_record(instance.geonet_id)
instance.geonet_id = None
instance.save()
except Exception as e:
messages.error(
request,
"La fiche de metadonnées a été supprimée avec succès.")
else:
messages.success(
request,
"La fiche de metadonnées a été supprimée avec succès.")
return redirect(reverse(self.namespace, id=instance.id))
root = ET.fromstring(request.body)
ns = {
'gmd': 'http://www.isotc211.org/2005/gmd',
'gco': 'http://www.isotc211.org/2005/gco'
}
id = root.find('gmd:fileIdentifier/gco:CharacterString', ns).text
record = ET.tostring(root,
encoding='utf-8',
method='xml',
short_empty_elements=True)
if not geonet.is_record_exists(id):
try:
geonet.create_record(id, record)
except Exception:
messages.error(
request,
'La création de la fiche de métadonnées a échoué.')
else:
geonet.publish(id) # Toujours publier la fiche
instance.geonet_id = id
instance.save()
messages.success(
request,
'La fiche de metadonnées a été créée avec succès.')
else:
try:
geonet.update_record(id, record)
except Exception:
messages.error(
request,
'La mise à jour de la fiche de métadonnées a échoué.')
else:
messages.success(
request,
'La fiche de metadonnées a été créée avec succès.')
return HttpResponse()
def get(self, request, id, *args, **kwargs):
user, profile = user_and_profile(request)
instance = get_object_or_404(Dataset, id=id)
config = {
'app_name': 'mdEdit',
'app_title': 'mdEdit',
'app_version': '0.14.9',
'app_copyrights': '(c) CIGAL 2016',
'languages': {'locales': ['fr']},
'defaultLanguage': 'fr',
'server_url_md': GEONETWORK_URL,
'views': {
'list': [{
# 'path': '{id}/edit/'.format(id=id),
'path': reverse('idgo_admin:dataset_mdedit_tpl_edit', kwargs={'id': instance.id}),
'values': {'fr': 'Edition'},
'locales': {'fr': join_url('views/edit/tpl-edit_fr.json')}
}, {
'path': join_url('tpl-view.html', path=MDEDIT_HTML_PATH),
'values': {'fr': 'Vue'},
'locales': {'fr': join_url('views/view/tpl-view_fr.json')}
}]},
'models': {
'list': [{
'path': join_url(MDEDIT_DATASET_MODEL),
'value': 'Modèle de fiche vierge'
}]},
'locales': MDEDIT_LOCALES,
'locales_path': join_url('locales/'),
'geographicextents_list': join_url('list_geographicextents.json'),
'referencesystems_list': join_url('list_referencesystems.json'),
'static_root': join_url('libs/mdedit/', path=STATIC_URL),
'modal_template': {
'help': join_url('modal-help.html', path=MDEDIT_HTML_PATH)}}
context = {'dataset': instance,
'doc_url': READTHEDOC_URL_INSPIRE,
'config': config}
record = instance.geonet_id and geonet.get_record(str(instance.geonet_id)) or None
if record:
xml = record.xml.decode(encoding='utf-8')
context['record_xml'] = re.sub('\n', '', xml).replace("'", "\\'") # C'est moche
else:
context['record_obj'] = prefill_dataset_model(instance)
return render_with_info_profile(request, self.template, context=context)
def get(self, request, dataset_name):
"""Voir la fiche de metadonnées du jeu de données."""
instance = None
for dataset in handler_get_request(request):
if dataset.slug == dataset_name:
instance = dataset
break
if not instance or (instance and not instance.geonet_id):
raise Http404()
try:
record = geonet.get_record(str(instance.geonet_id))
except Exception as e:
return JsonResponse({'error': e.__str__()}, status=400)
else:
return HttpResponse(record.xml, content_type='application/xml')
def put(self, request, dataset_name):
"""Enregistrer la ficher de métadonnées du jeu de données."""
request.PUT, _ = parse_request(request)
request.PUT._mutable = True
instance = None
for dataset in handler_get_request(request):
if dataset.slug == dataset_name:
instance = dataset
break
if not instance:
raise Http404()
root = ET.fromstring(request.PUT.get('xml'))
ns = {
'gmd': 'http://www.isotc211.org/2005/gmd',
'gco': 'http://www.isotc211.org/2005/gco'
}
geonet_id = root.find('gmd:fileIdentifier/gco:CharacterString',
ns).text
if not geonet_id:
return JsonResponse({'error': 'fileIdentifier not found'},
status=400)
record = ET.tostring(root,
encoding='utf-8',
method='xml',
short_empty_elements=True)
try:
if not geonet.get_record(geonet_id):
try:
geonet.create_record(geonet_id, record)
except Exception as e:
return JsonResponse({'error': e.__str__()}, status=400)
else:
geonet.publish(geonet_id)
dataset.geonet_id = geonet_id
else:
try:
geonet.update_record(geonet_id, record)
except Exception as e:
return JsonResponse({'error': e.__str__()}, status=400)
dataset.save(synchronize=True)
except GenericException as e:
return JsonResponse({'error': e.details}, status=400)
return HttpResponse(status=204)
def post(self, request, id, *args, **kwargs):
user, profile = user_and_profile(request)
dataset = get_object_or_404(Dataset, id=id)
if not request.is_ajax():
return HttpResponseRedirect(
reverse(self.namespace, kwargs={'dataset_id': id}))
root = ET.fromstring(request.body)
ns = {'gmd': 'http://www.isotc211.org/2005/gmd',
'gco': 'http://www.isotc211.org/2005/gco'}
id = root.find('gmd:fileIdentifier/gco:CharacterString', ns).text
record = ET.tostring(
root, encoding='utf-8', method='xml', short_empty_elements=True)
# Ça marche mais c'est moche et illisible...
# TODO: faire du code plus beau ; gérer les exceptions mieux que ça.
if not geonet.get_record(id):
try:
geonet.create_record(id, record)
except Exception:
messages.error(request, 'La création de la fiche de métadonnées a échoué.')
else:
geonet.publish(id) # Toujours publier la fiche
dataset.geonet_id = UUID(id)
dataset.save(current_user=None)
messages.success(
request, 'La fiche de metadonnées a été créée avec succès.')
else:
try:
geonet.update_record(id, record)
except Exception:
messages.error(request, 'La mise à jour de la fiche de métadonnées a échoué.')
else:
messages.success(
request, 'La fiche de metadonnées a été créée avec succès.')
return HttpResponse()
def save(self, *args, harvest=True, **kwargs):
Category = apps.get_model(app_label='idgo_admin',
model_name='Category')
Dataset = apps.get_model(app_label='idgo_admin', model_name='Dataset')
License = apps.get_model(app_label='idgo_admin', model_name='License')
Resource = apps.get_model(app_label='idgo_admin',
model_name='Resource')
ResourceFormats = apps.get_model(app_label='idgo_admin',
model_name='ResourceFormats')
# (1) Supprimer les jeux de données qui ne sont plus synchronisés
previous = self.pk and RemoteCsw.objects.get(pk=self.pk)
if previous:
for dataset in Dataset.harvested_csw.filter(
remote_instance=previous):
dataset.delete()
else:
# Dans le cas d'une création, on vérifie si l'URL CSW est valide
try:
with CswBaseHandler(self.url):
pass
except CswBaseError as e:
raise ValidationError(e.__str__(), code='url')
# (2) Sauver l'instance
super().save(*args, **kwargs)
# (3) Créer/Mettre à jour les jeux de données synchronisés
# On récupère dans le `stack` l'utilisateur effectuant l'opération
editor = None
for entry in inspect.stack():
try:
editor = entry[0].f_locals['request'].user._wrapped
except (KeyError, AttributeError):
continue
break
if not previous:
return
if harvest:
# Puis on moissonne le catalogue
try:
ckan_ids = []
geonet_ids = []
with transaction.atomic():
with CswBaseHandler(self.url) as csw:
packages = csw.get_packages(
xml=self.getrecords or None)
for package in packages:
if not package['type'] == 'dataset':
continue
geonet_id = package['id']
update_frequency = dict(Dataset.FREQUENCY_CHOICES).get(
package.get('frequency'), 'unknown')
update_frequency = package.get('frequency')
if not (update_frequency and update_frequency in dict(
Dataset.FREQUENCY_CHOICES).keys()):
update_frequency = 'unknown'
date_creation = package.get('dataset_creation_date',
None)
if date_creation:
try:
date_creation = datetime.strptime(
date_creation, ISOFORMAT_DATE)
except ValueError as e:
logger.warning(e)
date_creation = None
date_modification = package.get(
'dataset_modification_date', None)
if date_modification:
try:
date_modification = datetime.strptime(
date_modification, ISOFORMAT_DATE)
except ValueError as e:
logger.warning(e)
date_modification = None
date_publication = package.get(
'dataset_publication_date', None)
if date_publication:
try:
date_publication = datetime.strptime(
date_publication, ISOFORMAT_DATE)
except ValueError as e:
logger.warning(e)
date_publication = None
# Licence
license_titles = package.get('license_titles')
filters = [
Q(slug__in=license_titles),
Q(title__in=license_titles),
Q(alternate_titles__overlap=license_titles),
]
license = License.objects.filter(reduce(
ior, filters)).distinct().first()
if not license:
try:
license = License.objects.get(
slug=settings.DEFAULTS_VALUES.get(
'LICENSE'))
except License.DoesNotExist:
license = License.objects.first()
# On pousse la fiche de MD dans Geonet
if not geonet.get_record(geonet_id):
try:
geonet.create_record(geonet_id, package['xml'])
except Exception as e:
logger.warning(
'La création de la fiche de métadonnées a échoué.'
)
logger.error(e)
else:
geonet_ids.append(geonet_id)
geonet.publish(
geonet_id) # Toujours publier la fiche
else:
try:
geonet.update_record(geonet_id, package['xml'])
except Exception as e:
logger.warning(
'La mise à jour de la fiche de métadonnées a échoué.'
)
logger.error(e)
slug = 'sync{}-{}'.format(
str(uuid.uuid4())[:7].lower(),
slugify(geonet_id))[:100]
kvp = {
'slug':
slug,
'title':
package.get('title'),
'description':
package.get('notes'),
'date_creation':
date_creation and date_creation.date(),
'date_modification':
date_modification and date_modification.date(),
'date_publication':
date_publication and date_publication.date(),
'editor':
editor,
'license':
license,
'owner_email':
self.organisation.email or DEFAULT_CONTACT_EMAIL,
'owner_name':
self.organisation.legal_name
or DEFAULT_PLATFORM_NAME,
'organisation':
self.organisation,
'published':
not package.get('private'),
'remote_instance':
self,
'remote_dataset':
geonet_id,
'update_frequency':
update_frequency,
'bbox':
package.get('bbox'),
# broadcaster_email
# broadcaster_name
# data_type
# geocover
'geonet_id':
geonet_id,
# granularity
# thumbnail
# support
}
dataset, created = Dataset.harvested_csw.update_or_create(
**kvp)
if created:
ckan_ids.append(dataset.ckan_id)
categories_name = [
m['name'] for m in package.get('groups', [])
]
iso_topic_reverse = dict(
(v, k)
for k, v in Category._meta.fields[5].choices)
filters = [
Q(slug__in=categories_name),
Q(name__in=categories_name),
Q(iso_topic__in=[
m['name'] for m in package.get('groups', [])
]),
Q(iso_topic__in=[
iso_topic_reverse.get(name)
for name in categories_name
]),
Q(alternate_titles__overlap=categories_name),
]
categories = Category.objects.filter(
reduce(ior, filters)).distinct()
if categories:
dataset.categories.set(categories, clear=True)
if not created:
dataset.keywords.clear()
keywords = [
tag['display_name'] for tag in package.get('tags')
]
dataset.keywords.add(*keywords)
dataset.save(current_user=None,
synchronize=True,
activate=False)
for resource in package.get('resources', []):
try:
ckan_id = uuid.uuid4()
except ValueError as e:
logger.exception(e)
logger.error(
"I can't crash here, so I do not pay any attention to this error."
)
continue
filters = []
protocol = resource.get('protocol')
protocol and filters.append(Q(protocol=protocol))
mimetype = resource.get('mimetype')
mimetype and filters.append(
Q(mimetype__overlap=[mimetype]))
try:
format_type = ResourceFormats.objects.get(
reduce(iand, filters))
except (ResourceFormats.MultipleObjectsReturned,
ResourceFormats.DoesNotExist, TypeError):
format_type = None
kvp = {
'ckan_id': ckan_id,
'dataset': dataset,
'format_type': format_type,
'title': resource['name'] or resource['url'],
'referenced_url': resource['url'],
}
try:
resource = Resource.objects.get(
ckan_id=ckan_id)
except Resource.DoesNotExist:
resource = Resource.default.create(save_opts={
'current_user':
editor,
'synchronize':
True
},
**kvp)
else:
for k, v in kvp.items():
setattr(resource, k, v)
resource.save(current_user=editor,
synchronize=True)
except Exception as e:
for id in ckan_ids:
logger.warning(
'Delete CKAN package : {id}.'.format(id=str(id)))
CkanHandler.purge_dataset(str(id))
for id in geonet_ids:
logger.warning('Delete MD : {id}.'.format(id=str(id)))
geonet.delete_record(id)
logger.error(e)
raise CriticalError()
else:
for id in ckan_ids:
CkanHandler.publish_dataset(id=str(id), state='active')
def post(self, request, id, *args, **kwargs):
user = request.user
dataset = get_object_or_404(Dataset, id=id)
delete = 'delete' in request.POST
save = 'save' in request.POST
save_and_continue = 'continue' in request.POST
if delete and dataset.geonet_id:
try:
geonet.delete_record(dataset.geonet_id)
dataset.geonet_id = None
dataset.save(current_user=None)
except Exception:
messages.error(
request,
"La fiche de metadonnées a été supprimée avec succès.")
else:
messages.success(
request,
"La fiche de metadonnées a été supprimée avec succès.")
# finally:
return redirect('idgo_admin:dataset_mdedit', id=id)
if save or save_and_continue:
data = dict(request.POST)
dataset.title = data['dataTitle'][0] or dataset.title
dataset.description = data['dataAbstract'][0] or None
date_creation = data['dataDateCreation'][0] or None
if date_creation:
dataset.date_creation = datetime.strptime(
date_creation, "%Y-%m-%d").date()
date_modification = data['dataDateRevision'][0] or None
if date_modification:
dataset.date_modification = datetime.strptime(
date_modification, "%Y-%m-%d").date()
date_publication = data['dataDatePublication'][0] or None
if date_publication:
dataset.date_publication = datetime.strptime(
date_publication, "%Y-%m-%d").date()
dataset.update_frequency = {
'notPlanned': 'never', # [011]
'asNeeded': 'asneeded', # [009]
'irregular': 'intermittently', # [010]
'continual': 'continuously', # [001]
'daily': 'daily', # [002]
'weekly': 'weekly', # [003]
'fortnightly': 'fortnightly', # [004]
'monthly': 'monthly', # [005]
'quarterly': 'quaterly', # [006]
'semiannual': 'biannually', # [007]
'annual': 'annually' # [008]
}.get(data['dataMaintenanceFrequency'][0], 'unknown') # [012]
keywords = [
k.strip() for l in [s.split(',') for s in data['keyword']]
for k in l if k
]
if keywords:
dataset.keywords.clear()
for k in keywords:
dataset.keywords.add(k)
root = ET.fromstring(request.POST.get('xml'))
ns = {
'gmd': 'http://www.isotc211.org/2005/gmd',
'gco': 'http://www.isotc211.org/2005/gco'
}
geonet_id = root.find('gmd:fileIdentifier/gco:CharacterString',
ns).text
record = ET.tostring(root,
encoding='utf-8',
method='xml',
short_empty_elements=True)
error = False
if not geonet.get_record(geonet_id):
try:
geonet.create_record(geonet_id, record)
except Exception:
error = True
messages.error(
request,
"La création de la fiche de métadonnées a échoué.")
else:
# Toujours publier la fiche
geonet.publish(geonet_id)
dataset.geonet_id = geonet_id
messages.success(
request,
"La fiche de metadonnées a été créée avec succès.")
else:
try:
geonet.update_record(geonet_id, record)
except Exception:
error = True
messages.error(
request,
"La mise à jour de la fiche de métadonnées a échoué.")
else:
messages.success(
request,
"La fiche de metadonnées a été mise à jour avec succès."
)
if not error:
dataset.save(current_user=user, synchronize=True)
if save_and_continue:
return redirect('idgo_admin:dataset_mdedit', id=id)
else:
return redirect('idgo_admin:list_my_datasets')
def delete_attached_md(sender, instance, **kwargs):
if instance.geonet_id:
geonet.delete_record(instance.geonet_id)
logger.info("Dataset MD '%s' has been deleted." % instance.geonet_id)