def delete(self, *args, current_user=None, **kwargs): with_user = current_user # On supprime toutes les ressources attachées au jeu de données Resource = apps.get_model(app_label='idgo_admin', model_name='Resource') for resource in Resource.objects.filter(dataset=self): resource.delete(current_user=current_user, synchronize_dataset=False) # > > > > > > BETA < < < < < < # if BETA: ResourceBeta = apps.get_model(app_label='idgo_resource', model_name='Resource') for resource_beta in ResourceBeta.objects.filter(dataset=self): resource_beta._related.delete() resource_beta.delete() # > > > > > > BETA < < < < < < # # On supprime le package CKAN ckan_id = str(self.ckan_id) if with_user: username = with_user.username apikey = CkanHandler.get_user(username)['apikey'] with CkanUserHandler(apikey=apikey) as ckan_user: ckan_user.delete_dataset(ckan_id) else: CkanHandler.delete_dataset(ckan_id) CkanHandler.purge_dataset(ckan_id) # On supprime l'instance super().delete(*args, **kwargs)
def delete(self, *args, current_user=None, **kwargs): with_user = current_user # On supprime la ressource CKAN if with_user: username = with_user.username apikey = CkanHandler.get_user(username)['apikey'] with CkanUserHandler(apikey=apikey) as ckan_user: ckan_user.delete_resource(self.name) else: CkanHandler.delete_resource(self.name) # On supprime les ressources MRA try: MRAHandler.del_layer(self.name) ws_name = self.resource.dataset.organisation.slug if self.type == 'vector': MRAHandler.del_featuretype(ws_name, 'public', self.name) if self.type == 'raster': MRAHandler.del_coverage(ws_name, self.name, self.name) # MRAHandler.del_coveragestore(ws_name, self.name) except Exception as e: logger.error(e) pass # On supprime la table de données PostGIS try: drop_table(self.name) except Exception as e: logger.error(e) pass # Puis on supprime l'instance super().delete(*args, **kwargs)
def synchronize(self, with_user=None): """Synchronizer le jeu de données avec l'instance de CKAN.""" # 'with_user' n'est pas utiliser dans ce contexte # Définition des propriétés de la « ressource » # ============================================= id = self.name name = 'Aperçu cartographique'.format(name=self.resource.title) description = self.resource.description organisation = self.resource.dataset.organisation base_url = OWS_URL_PATTERN.format( organisation=organisation.slug).replace('?', '') getlegendgraphic = ( '{base_url}?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetLegendGraphic' '&LAYER={layer}&FORMAT=image/png').format(base_url=base_url, layer=id) api = { 'url': base_url, 'typename': id, 'getlegendgraphic': getlegendgraphic } try: DEFAULT_SRID = settings.DEFAULTS_VALUES['SRID'] except Exception: DEFAULT_SRID = 4326 else: SupportedCrs = apps.get_model(app_label='idgo_admin', model_name='SupportedCrs') try: SupportedCrs.objects.get(auth_name='EPSG', auth_code=DEFAULT_SRID) except SupportedCrs.DoesNotExist: DEFAULT_SRID = 4326 if self.type == 'vector': if self.resource.format_type.extension.lower() in ('json', 'geojson'): outputformat = 'shapezip' # Il faudrait être sûr que le format existe avec le même nom ! elif self.resource.format_type.extension.lower() in ('zip', 'tar'): outputformat = 'geojson' # Il faudrait être sûr que le format existe avec le même nom ! api[outputformat] = ( '{base_url}?SERVICE=WFS&VERSION=2.0.0&REQUEST=GetFeature' '&TYPENAME={typename}&OUTPUTFORMAT={outputformat}&CRSNAME=EPSG:{srid}' ).format(base_url=base_url, typename=id, outputformat=outputformat, srid=str(DEFAULT_SRID)) CkanHandler.update_resource(str(self.resource.ckan_id), api=json.dumps(api)) CkanHandler.push_resource_view(title=name, description=description, resource_id=str(self.resource.ckan_id), view_type='geo_view')
def handle(self, *args, **options): total = Resource.objects.all().count() count = 0 for resource in Resource.objects.all().order_by('id'): count += 1 logger.info("[%d/%d] - Check Resource '%d'." % (count, total, resource.pk)) if not resource.up_file: continue # else: url = urljoin(resource.ckan_url, 'download/%s' % resource.up_file.name) CkanHandler.update_resource(str(resource.ckan_id), url=url) try: CkanHandler.update_resource(str(resource.ckan_id), url=url) except Exception as e: logger.error("Error with Resource '%d' - '%s'." % (resource.pk, resource.title)) logger.exception(e) logger.warning("Continue...") logger.info("Updated Resource '%d' - '%s' (filename: '%s')." % (resource.pk, resource.title, resource.up_file.name))
def sync_ckan_allowed_users_by_resource(*args, **kwargs): """Synchroniser `ckan-restricted` pour le cas particuliers des autorisations par organisation. """ def get_all_users_for_organisations(organisation__in): filter = {'organisation__in': organisation__in, 'organisation__is_active': True} return [profile.user.username for profile in Profile.objects.filter(**filter)] for resource in Resource.objects.exclude(organisations_allowed=None): dataset = resource.dataset organisation__in = [r.pk for r in resource.organisations_allowed.all()] allowed_users = get_all_users_for_organisations(organisation__in) restricted = { 'allowed_users': ','.join(allowed_users), 'level': 'only_allowed_users'} ckan_params = { 'id': str(resource.ckan_id), 'restricted': json.dumps(restricted)} logger.info("Update 'restricted' for Resource '%d'" % resource.pk) try: package = CkanHandler.get_package(str(dataset.ckan_id)) CkanHandler.push_resource(package, **ckan_params) except Exception as e: logger.exception(e) logger.info("Continue...")
def delete(self, *args, current_user=None, synchronize_dataset=True, **kwargs): with_user = current_user for layer in self.get_layers(): layer.delete(current_user=current_user) # On supprime la ressource CKAN ckan_id = str(self.ckan_id) if with_user: username = with_user.username apikey = CkanHandler.get_user(username)['apikey'] with CkanUserHandler(apikey=apikey) as ckan_user: ckan_user.delete_resource(ckan_id) else: CkanHandler.delete_resource(ckan_id) # On supprime l'instance super().delete(*args, **kwargs) # Ce n'est vraiment pas une bonne idée de synchroniser ici le dataset : self.dataset.date_modification = timezone.now().date() self.dataset.save(current_user=current_user, synchronize=synchronize_dataset, update_fields=['date_modification'])
def handle_crige_partner(sender, instance, **kwargs): groupname = 'crige-partner' username = instance.user.username if CkanHandler.is_user_exists(username): if instance.crige_membership: CkanHandler.add_user_to_partner_group(username, groupname) else: CkanHandler.del_user_from_partner_group(username, groupname)
def post_save_organisation(sender, instance, **kwargs): # Mettre à jour en cascade les profiles (utilisateurs) Profile = apps.get_model(app_label='idgo_admin', model_name='Profile') for profile in Profile.objects.filter(organisation=instance): profile.crige_membership = instance.is_crige_partner profile.save() # Synchroniser avec l'organisation CKAN if CkanHandler.is_organisation_exists(str(instance.ckan_id)): CkanHandler.update_organisation(instance)
def save(self, commit=True, *args, **kwargs): user = super().save(commit=False) try: CkanHandler.update_user(user) except Exception as e: raise forms.ValidationError( "La modification de l'utilisateur sur CKAN a échoué: {}.".format(e)) if commit: user.save() return user
def merge_name(self, request, queryset_tags): datasets = Dataset.objects.filter( keywords__in=queryset_tags).distinct() if 'apply' in request.POST: form = NewKeywordForm(request.POST) if form.is_valid(): error = False name = form.cleaned_data.get('new_name') tag, created = Tag.objects.get_or_create(name=name) # WIP for dataset in datasets: dataset.keywords.add(tag) ckan_id = str(dataset.ckan_id) qs_dataset_keywords = dataset.keywords.all().exclude( id__in=queryset_tags) tags = [ *[{ 'name': k.name } for k in qs_dataset_keywords], *[{ 'name': tag.name }] ] logger.info('Update dataset %d with tags: %s' % (dataset.pk, tags)) try: CkanHandler.publish_dataset(id=ckan_id, tags=tags) except CkanBaseError as e: logger.exception(e) error = True dataset.keywords.remove(tag) break else: continue if error: messages.error(request, ( "Une erreur est survenue. " "Veuillez contacter l'administrateur de la plateforme." )) else: queryset_tags.exclude(pk=tag.pk).delete() messages.info( request, ("La mise à jour est effectuée avec succès.")) return HttpResponseRedirect(request.get_full_path()) else: # request.GET form = NewKeywordForm() # then template_html = 'admin/idgo_admin/taggit_merge_name.html' context = {'form': form, 'tags': queryset_tags, 'datasets': datasets} return render(request, template_html, context=context)
def post(self, request): try: user, profile = user_and_profile(request) except ProfileHttp404: return HttpResponseRedirect(reverse('server_cas:signIn')) form = UpdateAccountForm(request.POST, instance=user) if not form.is_valid(): return render_with_info_profile(request, self.template, context={'form': form}) try: with transaction.atomic(): if form.new_password: user.set_password(form.new_password) user.save() logout(request) login(request, user, backend='django.contrib.auth.backends.ModelBackend') for field in form.Meta.profile_fields: setattr(profile, field, form.cleaned_data[field]) profile.save() for field in form.Meta.user_fields: setattr(user, field, form.cleaned_data[field]) user.save() CkanHandler.update_user(user) except ValidationError as e: messages.error(request, e.message) return render_with_info_profile(request, self.template, context={'form': form}) except CkanBaseError as e: form.add_error('__all__', e.__str__()) messages.error(request, e.__str__()) return render_with_info_profile(request, self.template, context={'form': form}) messages.success(request, 'Votre compte a bien été mis à jour.') return render_with_info_profile(request, self.template, context={'form': form}, status=200)
def save(self, commit=True, *args, **kwargs): user = super().save(commit=False) pass_generated = self.password_generator() try: CkanHandler.add_user(user, pass_generated, state='active') except Exception as e: raise ValidationError( "L'ajout de l'utilisateur sur CKAN a échoué: {}.".format(e)) user.set_password(pass_generated) if commit: user.save() return user
def handle(self, *args, **options): queryset = Category.objects.all().order_by('id') total = queryset.count() count = 0 for instance in queryset: count += 1 logger.info("[%d/%d] - Save Resource %d." % (count, total, instance.pk)) if CkanHandler.is_group_exists(instance.slug): CkanHandler.update_group(instance) logger.info("'%s' is udpated." % instance.slug) else: CkanHandler.add_group(instance) logger.info("'%s' is created." % instance.slug)
def delete_ckan_resource(sender, instance, **kwargs): """Supprimer la resource CKAN à la suppression d'une resource.""" apikey = CkanHandler.get_user(instance.dataset.editor.username)['apikey'] with CkanUserHandler(apikey=apikey) as ckan: ckan.delete_resource(str(instance.ckan_id)) logger.info( "CKAN Resource \"{pk}\" has been deleted.".format(pk=instance.ckan_id))
def clean(self): username = self.cleaned_data.get('username') password = self.cleaned_data.get('password') if username and password: try: self.user = authenticate(request=self.request, username=username, password=password) except Exception as e: raise ValidationError(e.__str__()) if self.user is None: self.add_error('username', "Vérifiez votre nom d'utilisateur") self.add_error('password', 'Vérifiez votre mot de passe') raise ValidationError('User is not found') else: ckan_user = CkanHandler.get_user(username) if ckan_user and ckan_user['state'] == 'deleted': self.add_error('username', "Erreur interne d'authentification") raise ValidationError('CKAN user is deleted') if not self.user.is_active: self.add_error('username', "Ce compte n'est pas activé") raise ValidationError('User is not activate') return self.cleaned_data
def clean(self): if not self.unlock_terms and not self.cleaned_data.get( 'terms_and_conditions'): self.add_error( 'terms_and_conditions', "Vous devez accepter les conditions générales d'utilisation.") username = self.cleaned_data.get('username') if User.objects.filter(username=username).exists( ) or CkanHandler.is_user_exists(username): self.add_error('username', "Ce nom d'utilisateur est reservé.") email = self.cleaned_data.get('email') if User.objects.filter(email=email).exists(): self.add_error('email', 'Ce courriel est reservé.') password1 = self.cleaned_data.get('password1') password2 = self.cleaned_data.get('password2') if password1 and password2 and password1 != password2: self.add_error('password2', 'Vérifiez les mots de passes.') self.cleaned_data['password'] = self.cleaned_data.pop('password1') if IDGO_CONTRIBUTION_REDUCED_TO_PARTNERS: organisation = self.cleaned_data['organisation'] if organisation and not organisation.is_crige_partner: self.cleaned_data['contributor'] = False self.cleaned_data['referent'] = False return self.cleaned_data
def clean(self): # Vérifie la disponibilité du « slug » dans CKAN slug = self.slug or slugify(self.title)[:100] ckan_dataset = CkanHandler.get_package(slug) if ckan_dataset: if UUID(ckan_dataset['id'] ) != self.ckan_id and ckan_dataset['name'] == slug: raise ValidationError("L'URL du jeu de données est réservé.")
def asynchronous_tasks(filename, dir, content_type, data, user_pk, dataset_pk, resource_pk, package_id, **kwargs): if content_type.endswith('zip'): unzip(filename, dir, flush=True) else: raise NotImplementedError('TODO') username = User.objects.get(pk=user_pk).username apikey = CkanHandler.get_user(username)['apikey'] html = get_html_content(dataset_pk, resource_pk) upload = io.BytesIO(html.encode('utf-8')) data['upload'] = upload ckan_package = CkanHandler.get_package(package_id) apikey = CkanHandler.get_user(username)['apikey'] with CkanUserHandler(apikey=apikey) as ckan: ckan.publish_resource(ckan_package, **data) upload.close()
def synchronize(instance, with_user=None): ckan_package = CkanHandler.get_package(str(instance.dataset.ckan_id)) username = with_user and with_user.username or instance.dataset.editor.username apikey = CkanHandler.get_user(username)['apikey'] location = os.path.join(DIRECTORY_STORAGE, str(instance.pk)) base_url = reverse('resource:directory_storage', kwargs={ 'dataset_id': instance.dataset.pk, 'resource_id': instance.pk }) files = iterate(location, base_url=base_url) html = render_to_string('resource/store/ckan_resource_template.html', context={'files': files}) upload = io.BytesIO(html.encode('utf-8')) data = { 'id': str(instance.ckan_id), 'url': instance.store.url, 'name': instance.title, 'description': instance.description, 'lang': instance.language, 'data_type': instance.resource_type, 'view_type': 'text_view', 'upload': upload, 'size': '', 'mimetype': 'text/html', 'format': '', 'api': '{}', 'restricted_by_jurisdiction': '', 'extracting_service': 'False', 'crs': '', 'restricted': json.dumps({'level': 'public'}), } with CkanUserHandler(apikey=apikey) as ckan: ckan.publish_resource(ckan_package, **data)
def handle(self, *args, **options): dataset_qs = Dataset.default.filter( keywords__isnull=False).distinct().order_by('id') total = dataset_qs.count() count = 0 for dataset in dataset_qs: count += 1 qs_dataset_keywords = dataset.keywords.all() ckan_id = str(dataset.ckan_id) logger.info( "[%d/%d] - Synchronize Dataset %d (%s) with tags: '%s'." % (count, total, dataset.pk, ckan_id, "', '".join( [k.name for k in qs_dataset_keywords]))) try: CkanHandler.publish_dataset(id=ckan_id, tags=[{ 'name': k.name } for k in qs_dataset_keywords]) except Exception as e: logger.exception(e) logger.warning("Error was ingored.")
def save_ckan_resource(self, with_user=None): resource = self.instance if with_user: username = with_user.username else: username = resource.dataset.editor.username upload = open(self.filename, 'rb') data = { 'id': str(resource.ckan_id), 'url': '', 'name': resource.title, 'description': resource.description, 'lang': resource.language, 'data_type': resource.resource_type, 'upload': upload, 'size': resource.upload.file_path.size, 'format': resource.format_type.ckan_format, 'mimetype': resource.format_type.mimetype[0], 'view_type': resource.format_type.ckan_view, # 'api': '{}', 'restricted_by_jurisdiction': 'False', 'extracting_service': 'False', 'crs': '', 'restricted': json.dumps({'level': 'public'}), } ckan_package = CkanHandler.get_package(str(resource.dataset.ckan_id)) apikey = CkanHandler.get_user(username)['apikey'] with CkanUserHandler(apikey=apikey) as ckan: ckan.publish_resource(ckan_package, **data) upload.close()
def clean(self): if not self.unlock_terms and not self.cleaned_data.get('terms_and_conditions'): self.add_error('terms_and_conditions', "Vous devez accepter les conditions générales d'utilisation.") username = self.cleaned_data.get('username') if User.objects.filter(username=username).exists() or CkanHandler.is_user_exists(username): self.add_error('username', "Ce nom d'utilisateur est reservé.") email = self.cleaned_data.get('email') if User.objects.filter(email=email).exists(): self.add_error('email', 'Ce courriel est reservé.') password1 = self.cleaned_data.get('password1') password2 = self.cleaned_data.get('password2') if password1 and password2 and password1 != password2: self.add_error('password2', 'Vérifiez les mots de passes.') self.cleaned_data['password'] = self.cleaned_data.pop('password1') return self.cleaned_data
def handle(self, *args, **options): for resource in Resource.objects.exclude(organisations_allowed=None): dataset = resource.dataset ckan_params = { 'id': str(resource.ckan_id), 'restricted': json.dumps({ 'allowed_users': ','.join( get_all_users_for_organisations([ r.pk for r in resource.organisations_allowed.all() ])), 'level': 'only_allowed_users' }) } apikey = CkanHandler.get_user(dataset.editor.username)['apikey'] with CkanUserHandler(apikey=apikey) as ckan: package = ckan.get_package(str(dataset.ckan_id)) ckan.push_resource(package, **ckan_params)
def get_context(self, form, user, dataset): layer_rows = [] resource_rows = [] if dataset: for resource in Resource.objects.filter(dataset=dataset): resource_row_data = ( resource.pk, resource.title, resource.format_type.description if resource.format_type else None, resource.get_data_type_display(), resource.created_on.isoformat() if resource.created_on else None, resource.last_update.isoformat() if resource.last_update else None, resource.get_restricted_level_display(), str(resource.ckan_id), [layer.pk for layer in resource.get_layers()], resource.ogc_services, resource.extractable, ) resource_rows.append(resource_row_data) common = [ resource.pk, resource.title, resource.get_data_type_display(), resource.get_restricted_level_display(), resource.geo_restriction, resource.extractable, resource.ogc_services, ] layers = resource.get_layers() if layers: for layer in resource.get_layers(): layer_row_data = common.copy() layer_row_data.extend(( layer.pk, layer.mra_info['name'], layer.mra_info['title'], layer.mra_info['type'], layer.mra_info['enabled'], layer.mra_info['bbox'], layer.mra_info['attributes'], layer.mra_info['styles'], )) layer_rows.append(layer_row_data) licenses = [ (m.pk, m.license.pk) for m in LiaisonsContributeurs.get_contribs(profile=user.profile) if m.license] supports = [ (m.pk, {'name': m.name, 'email': m.email}) for m in Support.objects.all()] tags = CkanHandler.get_tags() return { 'target': target(dataset, user), 'dataset': dataset, 'form': form, 'licenses': dict(licenses), 'layer_rows': json.dumps(layer_rows), 'resource_rows': json.dumps(resource_rows), 'supports': json.dumps(dict(supports)), 'tags': json.dumps(tags), }
def save(self, *args, harvest=True, **kwargs): Category = apps.get_model(app_label='idgo_admin', model_name='Category') Dataset = apps.get_model(app_label='idgo_admin', model_name='Dataset') License = apps.get_model(app_label='idgo_admin', model_name='License') Resource = apps.get_model(app_label='idgo_admin', model_name='Resource') ResourceFormats = apps.get_model(app_label='idgo_admin', model_name='ResourceFormats') # (1) Supprimer les jeux de données qui ne sont plus synchronisés previous = self.pk and RemoteCsw.objects.get(pk=self.pk) if previous: for dataset in Dataset.harvested_csw.filter( remote_instance=previous): dataset.delete() else: # Dans le cas d'une création, on vérifie si l'URL CSW est valide try: with CswBaseHandler(self.url): pass except CswBaseError as e: raise ValidationError(e.__str__(), code='url') # (2) Sauver l'instance super().save(*args, **kwargs) # (3) Créer/Mettre à jour les jeux de données synchronisés # On récupère dans le `stack` l'utilisateur effectuant l'opération editor = None for entry in inspect.stack(): try: editor = entry[0].f_locals['request'].user._wrapped except (KeyError, AttributeError): continue break if not previous: return if harvest: # Puis on moissonne le catalogue try: ckan_ids = [] geonet_ids = [] with transaction.atomic(): with CswBaseHandler(self.url) as csw: packages = csw.get_packages( xml=self.getrecords or None) for package in packages: if not package['type'] == 'dataset': continue geonet_id = package['id'] update_frequency = dict(Dataset.FREQUENCY_CHOICES).get( package.get('frequency'), 'unknown') update_frequency = package.get('frequency') if not (update_frequency and update_frequency in dict( Dataset.FREQUENCY_CHOICES).keys()): update_frequency = 'unknown' date_creation = package.get('dataset_creation_date', None) if date_creation: try: date_creation = datetime.strptime( date_creation, ISOFORMAT_DATE) except ValueError as e: logger.warning(e) date_creation = None date_modification = package.get( 'dataset_modification_date', None) if date_modification: try: date_modification = datetime.strptime( date_modification, ISOFORMAT_DATE) except ValueError as e: logger.warning(e) date_modification = None date_publication = package.get( 'dataset_publication_date', None) if date_publication: try: date_publication = datetime.strptime( date_publication, ISOFORMAT_DATE) except ValueError as e: logger.warning(e) date_publication = None # Licence license_titles = package.get('license_titles') filters = [ Q(slug__in=license_titles), Q(title__in=license_titles), Q(alternate_titles__overlap=license_titles), ] license = License.objects.filter(reduce( ior, filters)).distinct().first() if not license: try: license = License.objects.get( slug=settings.DEFAULTS_VALUES.get( 'LICENSE')) except License.DoesNotExist: license = License.objects.first() # On pousse la fiche de MD dans Geonet if not geonet.get_record(geonet_id): try: geonet.create_record(geonet_id, package['xml']) except Exception as e: logger.warning( 'La création de la fiche de métadonnées a échoué.' ) logger.error(e) else: geonet_ids.append(geonet_id) geonet.publish( geonet_id) # Toujours publier la fiche else: try: geonet.update_record(geonet_id, package['xml']) except Exception as e: logger.warning( 'La mise à jour de la fiche de métadonnées a échoué.' ) logger.error(e) slug = 'sync{}-{}'.format( str(uuid.uuid4())[:7].lower(), slugify(geonet_id))[:100] kvp = { 'slug': slug, 'title': package.get('title'), 'description': package.get('notes'), 'date_creation': date_creation and date_creation.date(), 'date_modification': date_modification and date_modification.date(), 'date_publication': date_publication and date_publication.date(), 'editor': editor, 'license': license, 'owner_email': self.organisation.email or DEFAULT_CONTACT_EMAIL, 'owner_name': self.organisation.legal_name or DEFAULT_PLATFORM_NAME, 'organisation': self.organisation, 'published': not package.get('private'), 'remote_instance': self, 'remote_dataset': geonet_id, 'update_frequency': update_frequency, 'bbox': package.get('bbox'), # broadcaster_email # broadcaster_name # data_type # geocover 'geonet_id': geonet_id, # granularity # thumbnail # support } dataset, created = Dataset.harvested_csw.update_or_create( **kvp) if created: ckan_ids.append(dataset.ckan_id) categories_name = [ m['name'] for m in package.get('groups', []) ] iso_topic_reverse = dict( (v, k) for k, v in Category._meta.fields[5].choices) filters = [ Q(slug__in=categories_name), Q(name__in=categories_name), Q(iso_topic__in=[ m['name'] for m in package.get('groups', []) ]), Q(iso_topic__in=[ iso_topic_reverse.get(name) for name in categories_name ]), Q(alternate_titles__overlap=categories_name), ] categories = Category.objects.filter( reduce(ior, filters)).distinct() if categories: dataset.categories.set(categories, clear=True) if not created: dataset.keywords.clear() keywords = [ tag['display_name'] for tag in package.get('tags') ] dataset.keywords.add(*keywords) dataset.save(current_user=None, synchronize=True, activate=False) for resource in package.get('resources', []): try: ckan_id = uuid.uuid4() except ValueError as e: logger.exception(e) logger.error( "I can't crash here, so I do not pay any attention to this error." ) continue filters = [] protocol = resource.get('protocol') protocol and filters.append(Q(protocol=protocol)) mimetype = resource.get('mimetype') mimetype and filters.append( Q(mimetype__overlap=[mimetype])) try: format_type = ResourceFormats.objects.get( reduce(iand, filters)) except (ResourceFormats.MultipleObjectsReturned, ResourceFormats.DoesNotExist, TypeError): format_type = None kvp = { 'ckan_id': ckan_id, 'dataset': dataset, 'format_type': format_type, 'title': resource['name'] or resource['url'], 'referenced_url': resource['url'], } try: resource = Resource.objects.get( ckan_id=ckan_id) except Resource.DoesNotExist: resource = Resource.default.create(save_opts={ 'current_user': editor, 'synchronize': True }, **kvp) else: for k, v in kvp.items(): setattr(resource, k, v) resource.save(current_user=editor, synchronize=True) except Exception as e: for id in ckan_ids: logger.warning( 'Delete CKAN package : {id}.'.format(id=str(id))) CkanHandler.purge_dataset(str(id)) for id in geonet_ids: logger.warning('Delete MD : {id}.'.format(id=str(id))) geonet.delete_record(id) logger.error(e) raise CriticalError() else: for id in ckan_ids: CkanHandler.publish_dataset(id=str(id), state='active')
def save(self, *args, **kwargs): Category = apps.get_model(app_label='idgo_admin', model_name='Category') Dataset = apps.get_model(app_label='idgo_admin', model_name='Dataset') License = apps.get_model(app_label='idgo_admin', model_name='License') Resource = apps.get_model(app_label='idgo_admin', model_name='Resource') ResourceFormats = apps.get_model(app_label='idgo_admin', model_name='ResourceFormats') # (1) Supprimer les jeux de données qui ne sont plus synchronisés previous = self.pk and RemoteCkan.objects.get(pk=self.pk) if previous: remote_organisation__in = [ x for x in (previous.sync_with or []) if x not in (self.sync_with or []) ] filter = { 'remote_instance': previous, 'remote_organisation__in': remote_organisation__in, } # TODO: 'Dataset.harvested_ckan.filter(**filter).delete()' ne fonctionne pas for dataset in Dataset.harvested_ckan.filter(**filter): dataset.delete() else: # Dans le cas d'une création, on vérifie si l'URL CKAN est valide try: with CkanBaseHandler(self.url): pass except CkanBaseError as e: raise ValidationError(e.__str__(), code='url') # (2) Sauver l'instance super().save(*args, **kwargs) # (3) Créer/Mettre à jour les jeux de données synchronisés # On récupère dans le `stack` l'utilisateur effectuant l'opération editor = User.objects.get(pk=DEFAULT_USER_ID) for entry in inspect.stack(): try: editor = entry[0].f_locals['request'].user._wrapped except (KeyError, AttributeError): continue break # Puis on moissonne le catalogue if self.sync_with: try: ckan_ids = [] with transaction.atomic(): # TODO: Factoriser for value in self.sync_with: with CkanBaseHandler(self.url) as ckan: ckan_organisation = ckan.get_organisation( value, include_datasets=True, include_groups=True, include_tags=True) if not ckan_organisation.get('package_count', 0): continue for package in ckan_organisation.get('packages'): if not package['state'] == 'active' \ or not package['type'] == 'dataset': continue with CkanBaseHandler(self.url) as ckan: package = ckan.get_package(package['id']) ckan_id = uuid.UUID(package['id']) update_frequency = dict( Dataset.FREQUENCY_CHOICES).get( package.get('frequency'), 'unknown') update_frequency = package.get('frequency') if not (update_frequency and update_frequency in dict(Dataset.FREQUENCY_CHOICES).keys()): update_frequency = 'unknown' metadata_created = package.get( 'metadata_created', None) if metadata_created: metadata_created = datetime.strptime( metadata_created, ISOFORMAT_DATETIME) metadata_modified = package.get( 'metadata_modified', None) if metadata_modified: metadata_modified = datetime.strptime( metadata_modified, ISOFORMAT_DATETIME) try: mapping_licence = MappingLicence.objects.get( remote_ckan=self, slug=package.get('license_id')) except MappingLicence.DoesNotExist: try: license = License.objects.get( slug='other-at') except MappingLicence.DoesNotExist: license = None else: logger.warning("'{}' non trouvé".format( package.get('license_id'))) license = mapping_licence.licence slug = 'sync{}-{}'.format( str(uuid.uuid4())[:7].lower(), package.get('name'))[:100] kvp = { 'slug': slug, 'title': package.get('title'), 'description': package.get('notes'), 'date_creation': metadata_created and metadata_created.date(), 'date_modification': metadata_modified and metadata_modified.date(), # date_publication 'editor': editor, 'license': license, 'owner_email': self.organisation.email or DEFAULT_CONTACT_EMAIL, 'owner_name': self.organisation.legal_name or DEFAULT_PLATFORM_NAME, 'organisation': self.organisation, 'published': not package.get('private'), 'remote_instance': self, 'remote_dataset': ckan_id, 'remote_organisation': value, 'update_frequency': update_frequency, # bbox # broadcaster_email # broadcaster_name # data_type # geocover # geonet_id # granularity # thumbnail # support } dataset, created = Dataset.harvested_ckan.update_or_create( **kvp) mapping_categories = MappingCategory.objects.filter( remote_ckan=self, slug__in=[ m['name'] for m in package.get('groups', []) ]) if mapping_categories: dataset.categories = set( mc.category for mc in mapping_categories) if not created: dataset.keywords.clear() keywords = [ tag['display_name'] for tag in package.get('tags') ] dataset.keywords.add(*keywords) dataset.save(current_user=None, synchronize=True, activate=False) ckan_ids.append(dataset.ckan_id) for resource in package.get('resources', []): try: ckan_id = uuid.UUID(resource['id']) except ValueError as e: logger.exception(e) logger.error( "I can't crash here, so I do not pay any attention to this error." ) continue try: ckan_format = resource['format'].upper() format_type = ResourceFormats.objects.get( ckan_format=ckan_format) except (ResourceFormats. MultipleObjectsReturned, ResourceFormats.DoesNotExist, TypeError) as e: logger.exception(e) logger.error( "I can't crash here, so I do not pay any attention to this error." ) format_type = None kvp = { 'ckan_id': ckan_id, 'dataset': dataset, 'format_type': format_type, 'title': resource['name'], 'referenced_url': resource['url'], } try: resource = Resource.objects.get( ckan_id=ckan_id) except Resource.DoesNotExist: resource = Resource.default.create( save_opts={ 'current_user': None, 'synchronize': True }, **kvp) else: for k, v in kvp.items(): setattr(resource, k, v) resource.save(current_user=None, synchronize=True) except Exception as e: for id in ckan_ids: CkanHandler.purge_dataset(str(id)) logger.error(e) raise CriticalError() else: for id in ckan_ids: CkanHandler.publish_dataset(id=str(id), state='active')
def post_delete_organisation(sender, instance, **kwargs): if CkanHandler.is_organisation_exists(str(instance.ckan_id)): CkanHandler.purge_organisation(str(instance.ckan_id))
def post(self, request, *args, **kwargs): user, profile = user_and_profile(request) if not profile.crige_membership: raise Http404 context = self.get_context(request, user) footprint = request.POST.get('footprint') or None footprint = footprint and json.loads(footprint) layer_name = request.POST.get('layer') resource_name = request.POST.get('resource') dataset_name = request.POST.get('dataset') dst_crs = request.POST.get('crs') format_vector = request.POST.get('format-vector') or None if format_vector: dst_format_vector = ExtractorSupportedFormat.objects.get( name=format_vector, type='vector').details format_raster = request.POST.get('format-raster') or None if format_raster: dst_format_raster = ExtractorSupportedFormat.objects.get( name=format_raster, type='raster').details data_extractions = [] additional_files = [] if layer_name or resource_name: if layer_name: model = 'Layer' foreign_field = 'name' foreign_value = layer_name layer = get_object_or_404(Layer, name=layer_name) resource = layer.resource if resource_name: model = 'Resource' foreign_field = 'ckan_id' foreign_value = resource_name resource = get_object_or_404(Resource, ckan_id=resource_name) layer = resource.get_layers()[0] # Car relation 1-1 if layer.type == 'raster': data_extraction = { **{ 'source': layer.filename, }, **dst_format_raster } elif layer.type == 'vector': data_extraction = { **{ 'layer': layer.name, 'source': 'PG:host=postgis-master user=datagis dbname=datagis', }, **dst_format_vector } data_extraction['dst_srs'] = dst_crs or 'EPSG:2154' if resource.geo_restriction: footprint_restriction = \ json.loads(user.profile.organisation.jurisdiction.geom.geojson) if footprint: try: data_extraction['footprint'] = intersect( json.dumps(footprint), json.dumps(footprint_restriction)) except Exception as e: msg = "La zone d'extraction génère une erreur" messages.error(request, msg) return render_with_info_profile(request, self.template, context=context) else: data_extraction['footprint'] = footprint_restriction data_extraction['footprint_srs'] = 'EPSG:4326' elif footprint: data_extraction['footprint'] = footprint data_extraction['footprint_srs'] = 'EPSG:4326' data_extractions.append(data_extraction) # Pas d'`additional_files` dans le cas présent. elif dataset_name: model = 'Dataset' foreign_field = 'slug' foreign_value = dataset_name dataset = get_object_or_404(Dataset, slug=dataset_name) for resource in dataset.get_resources(): for layer in resource.get_layers(): if layer.type == 'raster': data_extraction = { **{ 'source': layer.filename, }, **dst_format_raster } elif layer.type == 'vector': data_extraction = { **{ 'layer': layer.name, 'source': 'PG:host=postgis-master user=datagis dbname=datagis', }, **dst_format_vector } data_extraction['dst_srs'] = dst_crs or 'EPSG:2154' if resource.geo_restriction: footprint_restriction = \ json.loads(user.profile.organisation.jurisdiction.geom.geojson) if footprint: data_extraction['footprint'] = intersect( json.dumps(footprint), json.dumps(footprint_restriction)) else: data_extraction[ 'footprint'] = footprint_restriction data_extraction['footprint_srs'] = 'EPSG:4326' elif footprint: data_extraction['footprint'] = footprint data_extraction['footprint_srs'] = 'EPSG:4326' data_extractions.append(data_extraction) if resource.data_type == 'annexe': additional_files.append({ 'file_name': resource.filename, 'dir_name': 'Documentation associée', 'file_location': CkanHandler.get_resource(str( resource.ckan_id)).get('url') }) query = { 'user_id': user.username, 'user_email_address': user.email, 'user_name': user.last_name, 'user_first_name': user.first_name, 'user_company': user.profile.organisation and user.profile.organisation.legal_name or '', 'user_address': user.profile.organisation and user.profile.organisation.full_address or '', 'data_extractions': data_extractions, 'additional_files': additional_files } r = requests.post(EXTRACTOR_URL, json=query) if r.status_code == 201: details = r.json() AsyncExtractorTask.objects.create( details=details, foreign_field=foreign_field, foreign_value=foreign_value, model=model, query=query, submission_datetime=details.get('submission_datetime'), uuid=UUID(details.get('task_id')), user=user) messages.success(request, ( "L'extraction a été ajoutée à la liste de tâche. " "Vous allez recevoir un e-mail une fois l'extraction réalisée." )) domain = Site.objects.get(name='extractor').domain url = 'http{secure}://{domain}{path}'.format( secure=request.is_secure and 's' or '', domain=domain, path=reverse('idgo_admin:extractor_dashboard')) return HttpResponseRedirect(url) else: if r.status_code == 400: details = r.json().get('detail') msg = '{}: {}'.format(details.get('title', 'Error'), ' '.join(details.get('list', 'Error'))) else: msg = "L'extracteur n'est pas disponible pour le moment." messages.error(request, msg) return render_with_info_profile(request, self.template, context=context)
def confirmation_mail(request, key): action = get_object_or_404(AccountActions, key=UUID(key), action='confirm_mail') if action.closed: message = 'Vous avez déjà validé votre adresse e-mail.' return render(request, 'idgo_admin/message.html', {'message': message}, status=200) user = action.profile.user profile = action.profile organisation = profile.organisation CkanHandler.activate_user(user.username) user.is_active = True action.profile.is_active = True user.save() action.profile.save() if organisation: # Demande de création d'une nouvelle organisation if not organisation.is_active: new_organisation_action = get_object_or_404( AccountActions, action='confirm_new_organisation', organisation=organisation, profile=profile, closed=None) url = request.build_absolute_uri( reverse('idgo_admin:confirm_new_orga', kwargs={'key': new_organisation_action.key})) send_organisation_creation_confirmation_mail( user, organisation, url) # Demande de rattachement (Profile-Organisation) rattachement_action = get_object_or_404(AccountActions, action='confirm_rattachement', organisation=organisation, profile=profile, closed=None) url = request.build_absolute_uri( reverse('idgo_admin:confirm_rattachement', kwargs={'key': rattachement_action.key})) send_membership_confirmation_mail(user, organisation, url) # Demande de rôle de référent en attente de validation try: LiaisonsReferents.objects.get(organisation=organisation, profile=profile, validated_on=None) except Exception: pass else: referent_action = get_object_or_404(AccountActions, action='confirm_referent', organisation=organisation, profile=profile, closed=None) url = request.build_absolute_uri( reverse('idgo_admin:confirm_referent', kwargs={'key': referent_action.key})) send_referent_confirmation_mail(user, organisation, url) # Demande de rôle de contributeur en attente de validation try: LiaisonsContributeurs.objects.get(profile=profile, organisation=organisation, validated_on=None) except Exception: pass else: contribution_action = get_object_or_404( AccountActions, action='confirm_contribution', organisation=organisation, profile=profile, closed=None) url = request.build_absolute_uri( reverse('idgo_admin:confirm_contribution', kwargs={'key': contribution_action.key})) send_contributor_confirmation_mail(user, organisation, url) send_successful_account_creation_mail(user) action.closed = timezone.now() action.save() message = ("Merci d'avoir confirmé votre adresse e-mail. " 'Toute demande de rattachement, contribution, ' 'ou rôle de référent technique pour une organisation, ' "ne sera effective qu'après validation " 'par un administrateur.') context = { 'message': message, 'button': { 'href': reverse('server_cas:signIn'), 'label': 'Se connecter' } } return render(request, 'idgo_admin/message.html', context, status=200)
def handle(self, request, *args, **kwargs): user = request.user if user.is_anonymous: profile = None else: try: profile = get_object_or_404(Profile, user=user) except Exception: raise ProfileHttp404 qs = request.POST or request.GET outputformat = qs.get('format') if not outputformat or outputformat not in ('odl', 'datasud'): raise Http404() if outputformat == 'odl': annotate = OrderedDict(( ('COLL_NOM', COLL_NOM), ('COLL_SIRET', COLL_SIRET), ('ID', ID), ('TITRE', TITRE), ('DESCRIPTION', DESCRIPTION), ('THEME', THEME), # ('DIFFUSEUR', DIFFUSEUR), ('PRODUCTEUR_NOM', PRODUCTEUR_NOM), ('PRODUCTEUR_SIRET', PRODUCTEUR_SIRET), ('COUV_SPAT_MAILLE', COUV_SPAT_MAILLE), ('COUV_SPAT_NOM', COUV_SPAT_NOM), ('COUV_TEMP_DEBUT', COUV_TEMP_DEBUT), ('COUV_TEMP_FIN', COUV_TEMP_DEBUT), ('DATE_PUBL', DATE_PUBL), ('FREQ_MAJ', FREQ_MAJ), ('DATE_MAJ', DATE_MAJ), ('MOTS_CLES', MOTS_CLES), ('LICENCE', LICENCE), ('NOMBRE_RESSOURCES', NOMBRE_RESSOURCES), ('FORMAT_RESSOURCES', FORMAT_RESSOURCES), # ('PROJECTION', PROJECTION), # ('LANG', LANG), ('URL', URL))) else: annotate = OrderedDict(( ('COLL_NOM', COLL_NOM), ('COLL_SIRET', COLL_SIRET), ('ID', ID), ('TITRE', TITRE), ('DESCRIPTION', DESCRIPTION), ('THEME', THEME), ('PRODUCTEUR_NOM', PRODUCTEUR_NOM), ('PRODUCTEUR_SIRET', PRODUCTEUR_SIRET), ('COUV_SPAT_MAILLE', COUV_SPAT_MAILLE), ('COUV_SPAT_NOM', COUV_SPAT_NOM), ('COUV_TEMP_DEBUT', COUV_TEMP_DEBUT), ('COUV_TEMP_FIN', COUV_TEMP_DEBUT), ('DATE_PUBL', DATE_PUBL), ('FREQ_MAJ', FREQ_MAJ), ('DATE_MAJ', DATE_MAJ), ('MOTS_CLES', MOTS_CLES), ('LICENCE', LICENCE), ('NOMBRE_RESSOURCES', NOMBRE_RESSOURCES), ('FORMAT_RESSOURCES', FORMAT_RESSOURCES), ('URL', URL), ('DATASUD_ID', DATASUD_ID), # ('DATASUD_MOT_CLES', DATASUD_MOT_CLES), # ('DATASUD_ORGA', DATASUD_ORGA), ('DATASUD_ORGA_ID', DATASUD_ORGA_ID), ('DATASUD_ORGA_URL', DATASUD_ORGA_URL), ('DATASUD_PRODUCTEUR_NOM', DATASUD_PRODUCTEUR_NOM), # ('DATASUD_PRODUCTEUR_EMAIL', DATASUD_PRODUCTEUR_EMAIL), ('DATASUD_DIFFUSEUR_NOM', DATASUD_DIFFUSEUR_NOM), # ('DATASUD_DIFFUSEUR_EMAIL', DATASUD_DIFFUSEUR_EMAIL), ('DATASUD_COUV_TERR', DATASUD_COUV_TERR), # ('DATASUD_INSPIRE', DATASUD_INSPIRE), # ('DATASUD_DATASET_URL', DATASUD_DATASET_URL), # ('DATASUD_INSPIRE_URL', DATASUD_INSPIRE_URL), ('DATASUD_DATE_CREATION', DATASUD_DATE_CREATION), # ('DATASUD_RESSOURCE_URLS', DATASUD_RESSOURCE_URLS), # ('DATASUD_RESSOURCE_TAILLE', DATASUD_RESSOURCE_TAILLE), ('DATASUD_RESSOURCE_TYPES', DATASUD_RESSOURCE_TYPES), ('DATASUD_DATASET_VUES', DATASUD_DATASET_VUES), ('DATASUD_RESSOURCES_TELECHARGEMENT', DATASUD_RESSOURCES_TELECHARGEMENT), ('DATASUD_DATASET_NOTE', DATASUD_DATASET_NOTE), ('DATASUD_DATASET_NB_NOTES', DATASUD_DATASET_NB_NOTES), ('DIFFUSEUR', DIFFUSEUR), ('PROJECTION', PROJECTION), ('LANG', LANG), )) values = list(annotate.keys()) if not profile: ids = qs.get('ids', '').split(',') datasets = Dataset.objects.filter( ckan_id__in=[UUID(id) for id in ids]) elif 'mode' in qs: mode = qs.get('mode') if mode == 'all': roles = profile.get_roles() if roles['is_admin']: QuerySet = Dataset.default.all() elif roles['is_referent']: kwargs = { 'profile': profile, 'validated_on__isnull': False } organisation__in = set( instance.organisation for instance in LiaisonsReferents.objects.filter( **kwargs)) filter = ior(Q(editor=user), Q(organisation__in=organisation__in)) QuerySet = Dataset.default.filter(filter) elif mode == 'mine': QuerySet = Dataset.default.filter(editor=user) elif mode == 'ckan_harvested': QuerySet = Dataset.harvested_ckan elif mode == 'csw_harvested': QuerySet = Dataset.harvested_csw else: raise Http404() datasets = get_filtered_datasets(QuerySet, qs) response = HttpResponse(content_type='text/csv') response[ 'Content-Disposition'] = 'attachment; filename=dataset_export.csv' response['Cache-Control'] = 'no-cache' writer = unicodecsv.writer(response, encoding='utf-8', quoting=csv.QUOTE_ALL, delimiter=',', quotechar='"') writer.writerow(values) for row in datasets.annotate(**annotate).values(*values): if not outputformat == 'odl': package = CkanHandler.get_package(str(row['ID']), include_tracking=True) dataset_view = 0 if 'tracking_summary' in package: dataset_view = package['tracking_summary'].get('total') row['DATASUD_DATASET_VUES'] = dataset_view resources_dl = 0 for resource in package.get('resources'): if 'tracking_summary' in resource: resources_dl += int( resource['tracking_summary'].get('total')) row['DATASUD_RESSOURCES_TELECHARGEMENT'] = resources_dl row['DATASUD_DATASET_NOTE'] = package.get('rating') row['DATASUD_DATASET_NB_NOTES'] = package.get('ratings_count') writer.writerow([row[value] for value in values]) return response