def post(self, request, *args, **kwargs):
user = request.user
if IDGO_FONCTIONAL_REDUCED_TO_PARTNERS and not user.profile.crige_membership:
raise Http404()
context = self.get_context(request, user)
footprint = request.POST.get('footprint') or None
footprint = footprint and json.loads(footprint)
layer_name = request.POST.get('layer')
resource_name = request.POST.get('resource')
if BETA:
resource_beta_name = request.POST.get('resource_beta')
dataset_name = request.POST.get('dataset')
dst_crs = request.POST.get('crs')
format_vector = request.POST.get('format-vector') or None
if format_vector:
dst_format_vector = ExtractorSupportedFormat.objects.get(
name=format_vector, type='vector').details
format_raster = request.POST.get('format-raster') or None
if format_raster:
dst_format_raster = ExtractorSupportedFormat.objects.get(
name=format_raster, type='raster').details
data_extractions = []
additional_files = []
if BETA and resource_beta_name:
model = 'Resource'
app_label = 'idgo_resource'
foreign_field = 'ckan_id'
foreign_value = resource_beta_name
resource_beta = get_object_or_404(ResourceBeta,
ckan_id=resource_beta_name)
layer_beta = resource_beta.geographiclayer
if layer_beta.mra_layer.config_json.get('type') == 'RASTER':
data_extraction = {
**{
'source': resource_beta._related.virtual_mosaic_file_path
},
**dst_format_raster
}
else:
pass # raise NotImplementedError
# TODO geo_restriction?
if footprint:
data_extraction['footprint'] = footprint
data_extraction['footprint_srs'] = 'EPSG:4326'
data_extractions.append(data_extraction)
elif layer_name or resource_name:
if layer_name:
model = 'Layer'
app_label = 'idgo_admin'
foreign_field = 'name'
foreign_value = layer_name
layer = get_object_or_404(Layer, name=layer_name)
resource = layer.resource
if resource_name:
model = 'Resource'
app_label = 'idgo_admin'
foreign_field = 'ckan_id'
foreign_value = resource_name
resource = get_object_or_404(Resource, ckan_id=resource_name)
layer = resource.get_layers()[0] # Car relation 1-1
if layer.type == 'raster':
data_extraction = {
**{
'source': layer.filename,
},
**dst_format_raster
}
elif layer.type == 'vector':
data_extraction = {
**{
'layer':
layer.name,
'source':
'PG:host={host} port={port} dbname={database} user={user} password={password}'.format(
host=IDGO_EXTRACTOR_DB_HOST,
port=IDGO_EXTRACTOR_DB_PORT,
database=IDGO_EXTRACTOR_DB_NAME,
user=IDGO_EXTRACTOR_DB_USERNAME,
password=IDGO_EXTRACTOR_DB_PASSWORD,
),
},
**dst_format_vector
}
data_extraction['dst_srs'] = dst_crs or 'EPSG:2154'
if resource_name and resource.geo_restriction:
footprint_restriction = json.loads(
user.profile.organisation.jurisdiction.geom.geojson)
if footprint:
try:
data_extraction['footprint'] = intersect(
json.dumps(footprint),
json.dumps(footprint_restriction))
except Exception:
msg = "La zone d'extraction génère une erreur"
messages.error(request, msg)
return render(request, self.template, context=context)
else:
data_extraction['footprint'] = footprint_restriction
data_extraction['footprint_srs'] = 'EPSG:4326'
elif footprint:
data_extraction['footprint'] = footprint
data_extraction['footprint_srs'] = 'EPSG:4326'
data_extractions.append(data_extraction)
elif dataset_name:
model = 'Dataset'
app_label = 'idgo_admin'
foreign_field = 'slug'
foreign_value = dataset_name
dataset = get_object_or_404(Dataset, slug=dataset_name)
for resource in dataset.get_resources():
for layer in resource.get_layers():
if layer.type == 'raster':
data_extraction = {
**{
'source': layer.filename,
},
**dst_format_raster
}
elif layer.type == 'vector':
data_extraction = {
**{
'layer':
layer.name,
'source':
'PG:host={host} port={port} dbname={database} user={user} password={password}'.format(
host=IDGO_EXTRACTOR_DB_HOST,
port=IDGO_EXTRACTOR_DB_PORT,
database=IDGO_EXTRACTOR_DB_NAME,
user=IDGO_EXTRACTOR_DB_USERNAME,
password=IDGO_EXTRACTOR_DB_PASSWORD,
),
},
**dst_format_vector
}
data_extraction['dst_srs'] = dst_crs or 'EPSG:2154'
if resource.geo_restriction:
footprint_restriction = json.loads(
user.profile.organisation.jurisdiction.geom.geojson
)
if footprint:
data_extraction['footprint'] = intersect(
json.dumps(footprint),
json.dumps(footprint_restriction))
else:
data_extraction[
'footprint'] = footprint_restriction
data_extraction['footprint_srs'] = 'EPSG:4326'
elif footprint:
data_extraction['footprint'] = footprint
data_extraction['footprint_srs'] = 'EPSG:4326'
data_extractions.append(data_extraction)
if resource.data_type == 'annexe':
additional_files.append({
'file_name':
resource.filename,
'dir_name':
'Documentation associée',
'file_location':
CkanHandler.get_resource(str(
resource.ckan_id)).get('url')
})
query = {
'user_id':
user.username,
'user_email_address':
user.email,
'user_name':
user.last_name,
'user_first_name':
user.first_name,
'user_company':
user.profile.organisation and user.profile.organisation.legal_name
or '',
'user_address':
user.profile.organisation
and user.profile.organisation.full_address or '',
'data_extractions':
data_extractions,
'additional_files':
additional_files,
}
r = requests.post(EXTRACTOR_URL, json=query)
if r.status_code == 201:
details = r.json()
AsyncExtractorTask.objects.create(
details=details,
foreign_field=foreign_field,
foreign_value=foreign_value,
model=model,
app_label=app_label,
query=query,
submission_datetime=details.get('submission_datetime'),
uuid=UUID(details.get('task_id')),
user=user)
messages.success(request, (
"L'extraction a été ajoutée à la liste de tâche. "
"Vous allez recevoir un e-mail une fois l'extraction réalisée."
))
return redirect('idgo_admin:extractor_dashboard')
else:
if r.status_code == 400:
details = r.json().get('detail')
try:
msg = '{}: {}'.format(
details.get('title', 'Error'),
' '.join(details.get('list', 'Error')))
except:
msg = str(details)
else:
msg = "L'extracteur n'est pas disponible pour le moment."
messages.error(request, msg)
return render(request, self.template, context=context)
def post(self, request, *args, **kwargs):
user, profile = user_and_profile(request)
if not profile.crige_membership:
raise Http404
context = self.get_context(request, user)
footprint = request.POST.get('footprint') or None
footprint = footprint and json.loads(footprint)
layer_name = request.POST.get('layer')
resource_name = request.POST.get('resource')
dataset_name = request.POST.get('dataset')
dst_crs = request.POST.get('crs')
format_vector = request.POST.get('format-vector') or None
if format_vector:
dst_format_vector = ExtractorSupportedFormat.objects.get(
name=format_vector, type='vector').details
format_raster = request.POST.get('format-raster') or None
if format_raster:
dst_format_raster = ExtractorSupportedFormat.objects.get(
name=format_raster, type='raster').details
data_extractions = []
additional_files = []
if layer_name or resource_name:
if layer_name:
model = 'Layer'
foreign_field = 'name'
foreign_value = layer_name
layer = get_object_or_404(Layer, name=layer_name)
resource = layer.resource
if resource_name:
model = 'Resource'
foreign_field = 'ckan_id'
foreign_value = resource_name
resource = get_object_or_404(Resource, ckan_id=resource_name)
layer = resource.get_layers()[0] # Car relation 1-1
if layer.type == 'raster':
data_extraction = {
**{
'source': layer.filename,
},
**dst_format_raster
}
elif layer.type == 'vector':
data_extraction = {
**{
'layer':
layer.name,
'source':
'PG:host=postgis-master user=datagis dbname=datagis',
},
**dst_format_vector
}
data_extraction['dst_srs'] = dst_crs or 'EPSG:2154'
if resource.geo_restriction:
footprint_restriction = \
json.loads(user.profile.organisation.jurisdiction.geom.geojson)
if footprint:
try:
data_extraction['footprint'] = intersect(
json.dumps(footprint),
json.dumps(footprint_restriction))
except Exception as e:
msg = "La zone d'extraction génère une erreur"
messages.error(request, msg)
return render_with_info_profile(request,
self.template,
context=context)
else:
data_extraction['footprint'] = footprint_restriction
data_extraction['footprint_srs'] = 'EPSG:4326'
elif footprint:
data_extraction['footprint'] = footprint
data_extraction['footprint_srs'] = 'EPSG:4326'
data_extractions.append(data_extraction)
# Pas d'`additional_files` dans le cas présent.
elif dataset_name:
model = 'Dataset'
foreign_field = 'slug'
foreign_value = dataset_name
dataset = get_object_or_404(Dataset, slug=dataset_name)
for resource in dataset.get_resources():
for layer in resource.get_layers():
if layer.type == 'raster':
data_extraction = {
**{
'source': layer.filename,
},
**dst_format_raster
}
elif layer.type == 'vector':
data_extraction = {
**{
'layer':
layer.name,
'source':
'PG:host=postgis-master user=datagis dbname=datagis',
},
**dst_format_vector
}
data_extraction['dst_srs'] = dst_crs or 'EPSG:2154'
if resource.geo_restriction:
footprint_restriction = \
json.loads(user.profile.organisation.jurisdiction.geom.geojson)
if footprint:
data_extraction['footprint'] = intersect(
json.dumps(footprint),
json.dumps(footprint_restriction))
else:
data_extraction[
'footprint'] = footprint_restriction
data_extraction['footprint_srs'] = 'EPSG:4326'
elif footprint:
data_extraction['footprint'] = footprint
data_extraction['footprint_srs'] = 'EPSG:4326'
data_extractions.append(data_extraction)
if resource.data_type == 'annexe':
additional_files.append({
'file_name':
resource.filename,
'dir_name':
'Documentation associée',
'file_location':
CkanHandler.get_resource(str(
resource.ckan_id)).get('url')
})
query = {
'user_id':
user.username,
'user_email_address':
user.email,
'user_name':
user.last_name,
'user_first_name':
user.first_name,
'user_company':
user.profile.organisation and user.profile.organisation.legal_name
or '',
'user_address':
user.profile.organisation
and user.profile.organisation.full_address or '',
'data_extractions':
data_extractions,
'additional_files':
additional_files
}
r = requests.post(EXTRACTOR_URL, json=query)
if r.status_code == 201:
details = r.json()
AsyncExtractorTask.objects.create(
details=details,
foreign_field=foreign_field,
foreign_value=foreign_value,
model=model,
query=query,
submission_datetime=details.get('submission_datetime'),
uuid=UUID(details.get('task_id')),
user=user)
messages.success(request, (
"L'extraction a été ajoutée à la liste de tâche. "
"Vous allez recevoir un e-mail une fois l'extraction réalisée."
))
domain = Site.objects.get(name='extractor').domain
url = 'http{secure}://{domain}{path}'.format(
secure=request.is_secure and 's' or '',
domain=domain,
path=reverse('idgo_admin:extractor_dashboard'))
return HttpResponseRedirect(url)
else:
if r.status_code == 400:
details = r.json().get('detail')
msg = '{}: {}'.format(details.get('title', 'Error'),
' '.join(details.get('list', 'Error')))
else:
msg = "L'extracteur n'est pas disponible pour le moment."
messages.error(request, msg)
return render_with_info_profile(request,
self.template,
context=context)
def synchronize(self,
url=None,
filename=None,
content_type=None,
file_extras=None,
with_user=None):
"""Synchronizer le jeu de données avec l'instance de CKAN."""
# Identifiant de la resource CKAN :
id = str(self.ckan_id)
ckan_resource = {}
try:
ckan_resource = CkanHandler.get_resource(id)
except Exception as e:
logger.warning(e)
# Définition des propriétés du « package » :
data = {
'crs': self.crs and self.crs.description or '',
'name': self.title,
'description': self.description,
'data_type': self.data_type,
'extracting_service': str(self.extractable or False), # I <3 CKAN
'format': self.format_type and self.format_type.ckan_format,
'view_type': self.format_type and self.format_type.ckan_view,
'id': id,
'lang': self.lang,
'restricted_by_jurisdiction': str(self.geo_restriction),
'url': url and url or '',
'api': ckan_resource.get('api', '{}'),
}
# (0) Aucune restriction
if self.restricted_level == 'public':
restricted = json.dumps({'level': 'public'})
# (1) Uniquement pour un utilisateur connecté
elif self.restricted_level == 'registered':
restricted = json.dumps({'level': 'registered'})
# (2) Seulement les utilisateurs indiquées
elif self.restricted_level == 'only_allowed_users':
restricted = json.dumps({
'allowed_users':
','.join(
self.profiles_allowed.exists()
and [p.user.username
for p in self.profiles_allowed.all()] or []),
'level':
'only_allowed_users'
})
# (3) Les utilisateurs de cette organisation
elif self.restricted_level == 'same_organization':
restricted = json.dumps({
'allowed_users':
','.join(
get_all_users_for_organisations(
self.organisations_allowed.all())),
'level':
'only_allowed_users'
})
# (3) Les utilisateurs des organisations indiquées
elif self.restricted_level == 'any_organization':
restricted = json.dumps({
'allowed_users':
','.join(
get_all_users_for_organisations(
self.organisations_allowed.all())),
'level':
'only_allowed_users'
})
# (4) Les utilisateurs partenaires IDGO
elif self.restricted_level == 'only_idgo_partners':
restricted = json.dumps({
'allowed_groups': ['idgo-partner'],
'level': 'only_group_member'
})
data['restricted'] = restricted
if self.referenced_url:
data['url'] = self.referenced_url
if self.dl_url and filename:
downloaded_file = File(open(filename, 'rb'))
data['upload'] = downloaded_file
data['size'] = downloaded_file.size
data['mimetype'] = content_type
if self.up_file and file_extras:
data['upload'] = self.up_file.file
data['size'] = file_extras.get('size')
data['mimetype'] = file_extras.get('mimetype')
if self.ftp_file:
if not url and filename:
data['upload'] = self.ftp_file.file
data['size'] = self.ftp_file.size
if url or filename:
if self.format_type and (
type(self.format_type.mimetype) is list
and len(self.format_type.mimetype)):
data['mimetype'] = self.format_type.mimetype[0]
else:
data['mimetype'] = 'text/plain'
# data['force_url_type'] = 'upload' # NON PREVU PAR CKAN API
if self.data_type == 'raw':
if self.ftp_file or self.dl_url or self.up_file:
data['resource_type'] = 'file.upload'
elif self.referenced_url:
data['resource_type'] = 'file'
if self.data_type == 'annexe':
data['resource_type'] = 'documentation'
if self.data_type == 'service':
data['resource_type'] = 'api'
ckan_package = CkanHandler.get_package(str(self.dataset.ckan_id))
if with_user:
username = with_user.username
apikey = CkanHandler.get_user(username)['apikey']
with CkanUserHandler(apikey=apikey) as ckan:
ckan.publish_resource(ckan_package, **data)
else:
return CkanHandler.publish_resource(ckan_package, **data)
