def post(self, request, *args, **kwargs): user = request.user if IDGO_FONCTIONAL_REDUCED_TO_PARTNERS and not user.profile.crige_membership: raise Http404() context = self.get_context(request, user) footprint = request.POST.get('footprint') or None footprint = footprint and json.loads(footprint) layer_name = request.POST.get('layer') resource_name = request.POST.get('resource') if BETA: resource_beta_name = request.POST.get('resource_beta') dataset_name = request.POST.get('dataset') dst_crs = request.POST.get('crs') format_vector = request.POST.get('format-vector') or None if format_vector: dst_format_vector = ExtractorSupportedFormat.objects.get( name=format_vector, type='vector').details format_raster = request.POST.get('format-raster') or None if format_raster: dst_format_raster = ExtractorSupportedFormat.objects.get( name=format_raster, type='raster').details data_extractions = [] additional_files = [] if BETA and resource_beta_name: model = 'Resource' app_label = 'idgo_resource' foreign_field = 'ckan_id' foreign_value = resource_beta_name resource_beta = get_object_or_404(ResourceBeta, ckan_id=resource_beta_name) layer_beta = resource_beta.geographiclayer if layer_beta.mra_layer.config_json.get('type') == 'RASTER': data_extraction = { **{ 'source': resource_beta._related.virtual_mosaic_file_path }, **dst_format_raster } else: pass # raise NotImplementedError # TODO geo_restriction? if footprint: data_extraction['footprint'] = footprint data_extraction['footprint_srs'] = 'EPSG:4326' data_extractions.append(data_extraction) elif layer_name or resource_name: if layer_name: model = 'Layer' app_label = 'idgo_admin' foreign_field = 'name' foreign_value = layer_name layer = get_object_or_404(Layer, name=layer_name) resource = layer.resource if resource_name: model = 'Resource' app_label = 'idgo_admin' foreign_field = 'ckan_id' foreign_value = resource_name resource = get_object_or_404(Resource, ckan_id=resource_name) layer = resource.get_layers()[0] # Car relation 1-1 if layer.type == 'raster': data_extraction = { **{ 'source': layer.filename, }, **dst_format_raster } elif layer.type == 'vector': data_extraction = { **{ 'layer': layer.name, 'source': 'PG:host={host} port={port} dbname={database} user={user} password={password}'.format( host=IDGO_EXTRACTOR_DB_HOST, port=IDGO_EXTRACTOR_DB_PORT, database=IDGO_EXTRACTOR_DB_NAME, user=IDGO_EXTRACTOR_DB_USERNAME, password=IDGO_EXTRACTOR_DB_PASSWORD, ), }, **dst_format_vector } data_extraction['dst_srs'] = dst_crs or 'EPSG:2154' if resource_name and resource.geo_restriction: footprint_restriction = json.loads( user.profile.organisation.jurisdiction.geom.geojson) if footprint: try: data_extraction['footprint'] = intersect( json.dumps(footprint), json.dumps(footprint_restriction)) except Exception: msg = "La zone d'extraction génère une erreur" messages.error(request, msg) return render(request, self.template, context=context) else: data_extraction['footprint'] = footprint_restriction data_extraction['footprint_srs'] = 'EPSG:4326' elif footprint: data_extraction['footprint'] = footprint data_extraction['footprint_srs'] = 'EPSG:4326' data_extractions.append(data_extraction) elif dataset_name: model = 'Dataset' app_label = 'idgo_admin' foreign_field = 'slug' foreign_value = dataset_name dataset = get_object_or_404(Dataset, slug=dataset_name) for resource in dataset.get_resources(): for layer in resource.get_layers(): if layer.type == 'raster': data_extraction = { **{ 'source': layer.filename, }, **dst_format_raster } elif layer.type == 'vector': data_extraction = { **{ 'layer': layer.name, 'source': 'PG:host={host} port={port} dbname={database} user={user} password={password}'.format( host=IDGO_EXTRACTOR_DB_HOST, port=IDGO_EXTRACTOR_DB_PORT, database=IDGO_EXTRACTOR_DB_NAME, user=IDGO_EXTRACTOR_DB_USERNAME, password=IDGO_EXTRACTOR_DB_PASSWORD, ), }, **dst_format_vector } data_extraction['dst_srs'] = dst_crs or 'EPSG:2154' if resource.geo_restriction: footprint_restriction = json.loads( user.profile.organisation.jurisdiction.geom.geojson ) if footprint: data_extraction['footprint'] = intersect( json.dumps(footprint), json.dumps(footprint_restriction)) else: data_extraction[ 'footprint'] = footprint_restriction data_extraction['footprint_srs'] = 'EPSG:4326' elif footprint: data_extraction['footprint'] = footprint data_extraction['footprint_srs'] = 'EPSG:4326' data_extractions.append(data_extraction) if resource.data_type == 'annexe': additional_files.append({ 'file_name': resource.filename, 'dir_name': 'Documentation associée', 'file_location': CkanHandler.get_resource(str( resource.ckan_id)).get('url') }) query = { 'user_id': user.username, 'user_email_address': user.email, 'user_name': user.last_name, 'user_first_name': user.first_name, 'user_company': user.profile.organisation and user.profile.organisation.legal_name or '', 'user_address': user.profile.organisation and user.profile.organisation.full_address or '', 'data_extractions': data_extractions, 'additional_files': additional_files, } r = requests.post(EXTRACTOR_URL, json=query) if r.status_code == 201: details = r.json() AsyncExtractorTask.objects.create( details=details, foreign_field=foreign_field, foreign_value=foreign_value, model=model, app_label=app_label, query=query, submission_datetime=details.get('submission_datetime'), uuid=UUID(details.get('task_id')), user=user) messages.success(request, ( "L'extraction a été ajoutée à la liste de tâche. " "Vous allez recevoir un e-mail une fois l'extraction réalisée." )) return redirect('idgo_admin:extractor_dashboard') else: if r.status_code == 400: details = r.json().get('detail') try: msg = '{}: {}'.format( details.get('title', 'Error'), ' '.join(details.get('list', 'Error'))) except: msg = str(details) else: msg = "L'extracteur n'est pas disponible pour le moment." messages.error(request, msg) return render(request, self.template, context=context)
def post(self, request, *args, **kwargs): user, profile = user_and_profile(request) if not profile.crige_membership: raise Http404 context = self.get_context(request, user) footprint = request.POST.get('footprint') or None footprint = footprint and json.loads(footprint) layer_name = request.POST.get('layer') resource_name = request.POST.get('resource') dataset_name = request.POST.get('dataset') dst_crs = request.POST.get('crs') format_vector = request.POST.get('format-vector') or None if format_vector: dst_format_vector = ExtractorSupportedFormat.objects.get( name=format_vector, type='vector').details format_raster = request.POST.get('format-raster') or None if format_raster: dst_format_raster = ExtractorSupportedFormat.objects.get( name=format_raster, type='raster').details data_extractions = [] additional_files = [] if layer_name or resource_name: if layer_name: model = 'Layer' foreign_field = 'name' foreign_value = layer_name layer = get_object_or_404(Layer, name=layer_name) resource = layer.resource if resource_name: model = 'Resource' foreign_field = 'ckan_id' foreign_value = resource_name resource = get_object_or_404(Resource, ckan_id=resource_name) layer = resource.get_layers()[0] # Car relation 1-1 if layer.type == 'raster': data_extraction = { **{ 'source': layer.filename, }, **dst_format_raster } elif layer.type == 'vector': data_extraction = { **{ 'layer': layer.name, 'source': 'PG:host=postgis-master user=datagis dbname=datagis', }, **dst_format_vector } data_extraction['dst_srs'] = dst_crs or 'EPSG:2154' if resource.geo_restriction: footprint_restriction = \ json.loads(user.profile.organisation.jurisdiction.geom.geojson) if footprint: try: data_extraction['footprint'] = intersect( json.dumps(footprint), json.dumps(footprint_restriction)) except Exception as e: msg = "La zone d'extraction génère une erreur" messages.error(request, msg) return render_with_info_profile(request, self.template, context=context) else: data_extraction['footprint'] = footprint_restriction data_extraction['footprint_srs'] = 'EPSG:4326' elif footprint: data_extraction['footprint'] = footprint data_extraction['footprint_srs'] = 'EPSG:4326' data_extractions.append(data_extraction) # Pas d'`additional_files` dans le cas présent. elif dataset_name: model = 'Dataset' foreign_field = 'slug' foreign_value = dataset_name dataset = get_object_or_404(Dataset, slug=dataset_name) for resource in dataset.get_resources(): for layer in resource.get_layers(): if layer.type == 'raster': data_extraction = { **{ 'source': layer.filename, }, **dst_format_raster } elif layer.type == 'vector': data_extraction = { **{ 'layer': layer.name, 'source': 'PG:host=postgis-master user=datagis dbname=datagis', }, **dst_format_vector } data_extraction['dst_srs'] = dst_crs or 'EPSG:2154' if resource.geo_restriction: footprint_restriction = \ json.loads(user.profile.organisation.jurisdiction.geom.geojson) if footprint: data_extraction['footprint'] = intersect( json.dumps(footprint), json.dumps(footprint_restriction)) else: data_extraction[ 'footprint'] = footprint_restriction data_extraction['footprint_srs'] = 'EPSG:4326' elif footprint: data_extraction['footprint'] = footprint data_extraction['footprint_srs'] = 'EPSG:4326' data_extractions.append(data_extraction) if resource.data_type == 'annexe': additional_files.append({ 'file_name': resource.filename, 'dir_name': 'Documentation associée', 'file_location': CkanHandler.get_resource(str( resource.ckan_id)).get('url') }) query = { 'user_id': user.username, 'user_email_address': user.email, 'user_name': user.last_name, 'user_first_name': user.first_name, 'user_company': user.profile.organisation and user.profile.organisation.legal_name or '', 'user_address': user.profile.organisation and user.profile.organisation.full_address or '', 'data_extractions': data_extractions, 'additional_files': additional_files } r = requests.post(EXTRACTOR_URL, json=query) if r.status_code == 201: details = r.json() AsyncExtractorTask.objects.create( details=details, foreign_field=foreign_field, foreign_value=foreign_value, model=model, query=query, submission_datetime=details.get('submission_datetime'), uuid=UUID(details.get('task_id')), user=user) messages.success(request, ( "L'extraction a été ajoutée à la liste de tâche. " "Vous allez recevoir un e-mail une fois l'extraction réalisée." )) domain = Site.objects.get(name='extractor').domain url = 'http{secure}://{domain}{path}'.format( secure=request.is_secure and 's' or '', domain=domain, path=reverse('idgo_admin:extractor_dashboard')) return HttpResponseRedirect(url) else: if r.status_code == 400: details = r.json().get('detail') msg = '{}: {}'.format(details.get('title', 'Error'), ' '.join(details.get('list', 'Error'))) else: msg = "L'extracteur n'est pas disponible pour le moment." messages.error(request, msg) return render_with_info_profile(request, self.template, context=context)
def synchronize(self, url=None, filename=None, content_type=None, file_extras=None, with_user=None): """Synchronizer le jeu de données avec l'instance de CKAN.""" # Identifiant de la resource CKAN : id = str(self.ckan_id) ckan_resource = {} try: ckan_resource = CkanHandler.get_resource(id) except Exception as e: logger.warning(e) # Définition des propriétés du « package » : data = { 'crs': self.crs and self.crs.description or '', 'name': self.title, 'description': self.description, 'data_type': self.data_type, 'extracting_service': str(self.extractable or False), # I <3 CKAN 'format': self.format_type and self.format_type.ckan_format, 'view_type': self.format_type and self.format_type.ckan_view, 'id': id, 'lang': self.lang, 'restricted_by_jurisdiction': str(self.geo_restriction), 'url': url and url or '', 'api': ckan_resource.get('api', '{}'), } # (0) Aucune restriction if self.restricted_level == 'public': restricted = json.dumps({'level': 'public'}) # (1) Uniquement pour un utilisateur connecté elif self.restricted_level == 'registered': restricted = json.dumps({'level': 'registered'}) # (2) Seulement les utilisateurs indiquées elif self.restricted_level == 'only_allowed_users': restricted = json.dumps({ 'allowed_users': ','.join( self.profiles_allowed.exists() and [p.user.username for p in self.profiles_allowed.all()] or []), 'level': 'only_allowed_users' }) # (3) Les utilisateurs de cette organisation elif self.restricted_level == 'same_organization': restricted = json.dumps({ 'allowed_users': ','.join( get_all_users_for_organisations( self.organisations_allowed.all())), 'level': 'only_allowed_users' }) # (3) Les utilisateurs des organisations indiquées elif self.restricted_level == 'any_organization': restricted = json.dumps({ 'allowed_users': ','.join( get_all_users_for_organisations( self.organisations_allowed.all())), 'level': 'only_allowed_users' }) # (4) Les utilisateurs partenaires IDGO elif self.restricted_level == 'only_idgo_partners': restricted = json.dumps({ 'allowed_groups': ['idgo-partner'], 'level': 'only_group_member' }) data['restricted'] = restricted if self.referenced_url: data['url'] = self.referenced_url if self.dl_url and filename: downloaded_file = File(open(filename, 'rb')) data['upload'] = downloaded_file data['size'] = downloaded_file.size data['mimetype'] = content_type if self.up_file and file_extras: data['upload'] = self.up_file.file data['size'] = file_extras.get('size') data['mimetype'] = file_extras.get('mimetype') if self.ftp_file: if not url and filename: data['upload'] = self.ftp_file.file data['size'] = self.ftp_file.size if url or filename: if self.format_type and ( type(self.format_type.mimetype) is list and len(self.format_type.mimetype)): data['mimetype'] = self.format_type.mimetype[0] else: data['mimetype'] = 'text/plain' # data['force_url_type'] = 'upload' # NON PREVU PAR CKAN API if self.data_type == 'raw': if self.ftp_file or self.dl_url or self.up_file: data['resource_type'] = 'file.upload' elif self.referenced_url: data['resource_type'] = 'file' if self.data_type == 'annexe': data['resource_type'] = 'documentation' if self.data_type == 'service': data['resource_type'] = 'api' ckan_package = CkanHandler.get_package(str(self.dataset.ckan_id)) if with_user: username = with_user.username apikey = CkanHandler.get_user(username)['apikey'] with CkanUserHandler(apikey=apikey) as ckan: ckan.publish_resource(ckan_package, **data) else: return CkanHandler.publish_resource(ckan_package, **data)