def create_files(names, zipped=False):
tmpdir = mkdtemp()
names = [os.path.join(tmpdir, f) for f in names]
for f in names:
# required for windows to read the shapefile in binary mode and the zip
# in non-binary
if zipped:
open(f, 'w').close()
else:
try:
open(f, 'wb').close()
except OSError:
# windows fails at writing special characters
# need to do something better here
print("Test does not work in Windows")
if zipped:
basefile = os.path.join(tmpdir, 'files.zip')
zf = zipfile.ZipFile(basefile, 'w', allowZip64=True)
with zf:
for f in names:
zf.write(f, os.path.basename(f))
for f in names:
os.unlink(f)
names = [basefile]
yield names
def _fixup_base_file(absolute_base_file, tempdir=None):
tempdir_was_created = False
if not tempdir or not os.path.exists(tempdir):
tempdir = mkdtemp()
tempdir_was_created = True
try:
if not os.path.isfile(absolute_base_file):
tmp_files = [f for f in os.listdir(tempdir) if os.path.isfile(os.path.join(tempdir, f))]
for f in tmp_files:
if zipfile.is_zipfile(os.path.join(tempdir, f)):
absolute_base_file = unzip_file(os.path.join(tempdir, f), '.shp', tempdir=tempdir)
absolute_base_file = os.path.join(tempdir,
absolute_base_file)
elif zipfile.is_zipfile(absolute_base_file):
absolute_base_file = unzip_file(absolute_base_file,
'.shp', tempdir=tempdir)
absolute_base_file = os.path.join(tempdir,
absolute_base_file)
if os.path.exists(absolute_base_file):
return absolute_base_file
else:
raise Exception(_(f'File does not exist: {absolute_base_file}'))
finally:
if tempdir_was_created:
# Get rid if temporary files that have been uploaded via Upload form
logger.debug(f"... Cleaning up the temporary folders {tempdir}")
shutil.rmtree(tempdir, ignore_errors=True)
def transfer_remote_file(self, temporary_folder=None):
def file_chunks_iterable(file, chunk_size=None):
"""
Read the file and yield chunks of ``chunk_size`` bytes (defaults to
``DEFAULT_BUFFER_CHUNK_SIZE``).
"""
chunk_size = chunk_size or settings.DEFAULT_BUFFER_CHUNK_SIZE
try:
file.seek(0)
except (AttributeError, io.UnsupportedOperation):
pass
while True:
data = file.read(chunk_size)
if not data:
break
yield data
self.temporary_folder = temporary_folder or mkdtemp()
self.file_path = os.path.join(self.temporary_folder, self.name)
if self._is_django_form_file:
with open(self.file_path, "wb") as tmp_file:
for chunk in self._django_form_file.chunks():
tmp_file.write(chunk)
else:
with open(self.file_path, "wb") as tmp_file, smart_open.open(
uri=self._original_file_uri, mode="rb") as original_file:
for chunk in file_chunks_iterable(original_file):
tmp_file.write(chunk)
return self.file_path
def write_files(self):
absolute_base_file = None
tempdir = mkdtemp()
if zipfile.is_zipfile(self.cleaned_data['base_file']):
absolute_base_file = unzip_file(self.cleaned_data['base_file'],
'.shp',
tempdir=tempdir)
else:
for field in self.spatial_files:
f = self.cleaned_data[field]
if f is not None:
path = os.path.join(tempdir, f.name)
with open(path, 'wb') as writable:
for c in f.chunks():
writable.write(c)
absolute_base_file = os.path.join(
tempdir, self.cleaned_data["base_file"].name)
return tempdir, absolute_base_file
def form_valid(self, form):
"""
If the form is valid, save the associated model.
"""
doc_form = form.cleaned_data
file = doc_form.pop('doc_file', None)
if file:
tempdir = mkdtemp()
dirname = os.path.basename(tempdir)
filepath = storage_manager.save(f"{dirname}/{file.name}", file)
storage_path = storage_manager.path(filepath)
self.object = resource_manager.update(self.object.uuid,
instance=self.object,
vals=dict(
owner=self.request.user,
files=[storage_path]))
if tempdir != os.path.dirname(storage_path):
shutil.rmtree(tempdir, ignore_errors=True)
register_event(self.request, EventType.EVENT_CHANGE, self.object)
url = hookset.document_detail_url(self.object)
return HttpResponseRedirect(url)
def get_files(filename):
"""Converts the data to Shapefiles or Geotiffs and returns
a dictionary with all the required files
"""
files = {}
# Verify if the filename is in ascii format.
try:
filename.encode('ascii')
except UnicodeEncodeError:
msg = f"Please use only characters from the english alphabet for the filename. '{os.path.basename(filename).encode('UTF-8', 'strict')}' is not yet supported."
raise GeoNodeException(msg)
# Let's unzip the filname in case it is a ZIP file
from geonode.utils import unzip_file, mkdtemp
tempdir = None
if is_zipfile(filename):
tempdir = mkdtemp()
_filename = unzip_file(filename,
'.shp', tempdir=tempdir)
if not _filename:
# We need to iterate files as filename could be the zipfile
import ntpath
from geonode.upload.utils import _SUPPORTED_EXT
file_basename, file_ext = ntpath.splitext(filename)
for item in os.listdir(tempdir):
item_basename, item_ext = ntpath.splitext(item)
if ntpath.basename(item_basename) == ntpath.basename(file_basename) and (
item_ext.lower() in _SUPPORTED_EXT):
filename = os.path.join(tempdir, item)
break
else:
filename = _filename
# Make sure the file exists.
if not os.path.exists(filename):
msg = f'Could not open {filename}. Make sure you are using a valid file'
logger.debug(msg)
raise GeoNodeException(msg)
base_name, extension = os.path.splitext(filename)
# Replace special characters in filenames - []{}()
glob_name = re.sub(r'([\[\]\(\)\{\}])', r'[\g<1>]', base_name)
if extension.lower() == '.shp':
required_extensions = dict(
shp='.[sS][hH][pP]', dbf='.[dD][bB][fF]', shx='.[sS][hH][xX]')
for ext, pattern in required_extensions.items():
matches = glob.glob(glob_name + pattern)
if len(matches) == 0:
msg = (f'Expected helper file {base_name}.{ext} does not exist; a Shapefile '
'requires helper files with the following extensions: '
f'{list(required_extensions.keys())}')
raise GeoNodeException(msg)
elif len(matches) > 1:
msg = ('Multiple helper files for %s exist; they need to be '
'distinct by spelling and not just case.') % filename
raise GeoNodeException(msg)
else:
files[ext] = matches[0]
matches = glob.glob(f"{glob_name}.[pP][rR][jJ]")
if len(matches) == 1:
files['prj'] = matches[0]
elif len(matches) > 1:
msg = ('Multiple helper files for %s exist; they need to be '
'distinct by spelling and not just case.') % filename
raise GeoNodeException(msg)
elif extension.lower() in cov_exts:
files[extension.lower().replace('.', '')] = filename
# Only for GeoServer
if check_ogc_backend(geoserver.BACKEND_PACKAGE):
matches = glob.glob(f"{os.path.dirname(glob_name)}.[sS][lL][dD]")
if len(matches) == 1:
files['sld'] = matches[0]
else:
matches = glob.glob(f"{glob_name}.[sS][lL][dD]")
if len(matches) == 1:
files['sld'] = matches[0]
elif len(matches) > 1:
msg = ('Multiple style files (sld) for %s exist; they need to be '
'distinct by spelling and not just case.') % filename
raise GeoNodeException(msg)
matches = glob.glob(f"{glob_name}.[xX][mM][lL]")
# shapefile XML metadata is sometimes named base_name.shp.xml
# try looking for filename.xml if base_name.xml does not exist
if len(matches) == 0:
matches = glob.glob(f"{filename}.[xX][mM][lL]")
if len(matches) == 1:
files['xml'] = matches[0]
elif len(matches) > 1:
msg = ('Multiple XML files for %s exist; they need to be '
'distinct by spelling and not just case.') % filename
raise GeoNodeException(msg)
return files, tempdir
def dataset_upload_metadata(request):
out = {}
errormsgs = []
form = NewLayerUploadForm(request.POST, request.FILES)
if form.is_valid():
tempdir = mkdtemp()
relevant_files = select_relevant_files(
['xml'],
iter(request.FILES.values())
)
logger.debug(f"relevant_files: {relevant_files}")
write_uploaded_files_to_disk(tempdir, relevant_files)
base_file = os.path.join(tempdir, form.cleaned_data["base_file"].name)
name = form.cleaned_data['dataset_title']
layer = Dataset.objects.filter(typename=name)
if layer.exists():
dataset_uuid, vals, regions, keywords, _ = parse_metadata(
open(base_file).read())
if dataset_uuid and layer.first().uuid != dataset_uuid:
out['success'] = False
out['errors'] = "The UUID identifier from the XML Metadata, is different from the one saved"
return HttpResponse(
json.dumps(out),
content_type='application/json',
status=404)
updated_dataset = update_resource(layer.first(), base_file, regions, keywords, vals)
updated_dataset.save()
out['status'] = ['finished']
out['url'] = updated_dataset.get_absolute_url()
out['bbox'] = updated_dataset.bbox_string
out['crs'] = {
'type': 'name',
'properties': updated_dataset.srid
}
out['ogc_backend'] = settings.OGC_SERVER['default']['BACKEND']
if hasattr(updated_dataset, 'upload_session'):
upload_session = updated_dataset.upload_session
upload_session.processed = True
upload_session.save()
status_code = 200
out['success'] = True
return HttpResponse(
json.dumps(out),
content_type='application/json',
status=status_code)
else:
out['success'] = False
out['errors'] = "Dataset selected does not exists"
status_code = 404
return HttpResponse(
json.dumps(out),
content_type='application/json',
status=status_code)
else:
for e in form.errors.values():
errormsgs.extend([escape(v) for v in e])
out['errors'] = form.errors
out['errormsgs'] = errormsgs
return HttpResponse(
json.dumps(out),
content_type='application/json',
status=500)
def form_valid(self, form):
"""
If the form is valid, save the associated model.
"""
doc_form = form.cleaned_data
file = doc_form.pop('doc_file', None)
if file:
tempdir = mkdtemp()
dirname = os.path.basename(tempdir)
filepath = storage_manager.save(f"{dirname}/{file.name}", file)
storage_path = storage_manager.path(filepath)
self.object = resource_manager.create(
None,
resource_type=Document,
defaults=dict(owner=self.request.user,
doc_url=doc_form.pop('doc_url', None),
title=doc_form.pop('title', file.name),
files=[storage_path]))
if tempdir != os.path.dirname(storage_path):
shutil.rmtree(tempdir, ignore_errors=True)
else:
self.object = resource_manager.create(
None,
resource_type=Document,
defaults=dict(owner=self.request.user,
doc_url=doc_form.pop('doc_url', None),
title=doc_form.pop('title', None)))
if settings.ADMIN_MODERATE_UPLOADS:
self.object.is_approved = False
self.object.was_approved = False
if settings.RESOURCE_PUBLISHING:
self.object.is_published = False
self.object.was_published = False
resource_manager.set_permissions(
None,
instance=self.object,
permissions=form.cleaned_data["permissions"],
created=True)
abstract = None
date = None
regions = []
keywords = []
bbox = None
url = hookset.document_detail_url(self.object)
out = {'success': False}
if getattr(settings, 'EXIF_ENABLED', False):
try:
from geonode.documents.exif.utils import exif_extract_metadata_doc
exif_metadata = exif_extract_metadata_doc(self.object)
if exif_metadata:
date = exif_metadata.get('date', None)
keywords.extend(exif_metadata.get('keywords', []))
bbox = exif_metadata.get('bbox', None)
abstract = exif_metadata.get('abstract', None)
except Exception:
logger.debug("Exif extraction failed.")
resource_manager.update(
self.object.uuid,
instance=self.object,
keywords=keywords,
regions=regions,
vals=dict(abstract=abstract,
date=date,
date_type="Creation",
bbox_polygon=BBOXHelper.from_xy(bbox).as_polygon()
if bbox else None),
notify=True)
resource_manager.set_thumbnail(self.object.uuid,
instance=self.object,
overwrite=False)
register_event(self.request, EventType.EVENT_UPLOAD, self.object)
if self.request.GET.get('no__redirect', False):
out['success'] = True
out['url'] = url
if out['success']:
status_code = 200
else:
status_code = 400
return HttpResponse(json.dumps(out),
content_type='application/json',
status=status_code)
else:
return HttpResponseRedirect(url)
def transfer_remote_files(self):
self.temporary_folder = mkdtemp()
for name, data_item_retriever in self.data_items.items():
file_path = data_item_retriever.transfer_remote_file(
self.temporary_folder)
self.file_paths[name] = file_path