def _extract_zip(zip_handler, destination, charset):
file_names = zip_handler.namelist()
zip_handler.extractall(destination)
paths = []
for p in file_names:
_f = os.path.join(destination, p)
fixup_shp_columnnames(_f, charset)
paths.append(_f)
return paths
def _extract_zip(zip_handler, destination, charset):
file_names = zip_handler.namelist()
zip_handler.extractall(destination)
paths = []
for p in file_names:
_f = os.path.join(destination, p)
fixup_shp_columnnames(_f, charset)
paths.append(_f)
return paths
def scan_file(file_name, scan_hint=None, charset=None):
'''get a list of SpatialFiles for the provided file'''
if not os.path.exists(file_name):
raise Exception(
_("Could not access to uploaded data.").encode('UTF-8'))
dirname = os.path.dirname(file_name)
if zipfile.is_zipfile(file_name):
paths, kept_zip = _process_zip(file_name,
dirname,
scan_hint=scan_hint,
charset=charset)
archive = file_name if kept_zip else None
else:
paths = []
for p in os.listdir(dirname):
_f = os.path.join(dirname, p)
fixup_shp_columnnames(_f, charset)
paths.append(_f)
archive = None
if paths is not None:
safe_paths = _rename_files(paths)
else:
safe_paths = []
found = []
for file_type in types:
for path in safe_paths:
path_extension = os.path.splitext(path)[-1][1:]
hint_ok = (scan_hint is None or file_type.code == scan_hint
or scan_hint in file_type.aliases)
if file_type.matches(path_extension) and hint_ok:
_f = file_type.build_spatial_file(path, safe_paths)
found_paths = [f.base_file for f in found]
if path not in found_paths:
found.append(_f)
# detect xmls and assign if a single upload is found
xml_files = _find_file_type(safe_paths, extension='.xml')
if xml_files:
if len(found) == 1:
found[0].xml_files = xml_files
else:
raise Exception(
_("One or more XML files was provided, but no " +
"matching files were found for them."))
# detect slds and assign if a single upload is found
sld_files = _find_file_type(safe_paths, extension='.sld')
if sld_files:
if len(found) == 1:
found[0].sld_files = sld_files
else:
raise Exception(
_("One or more SLD files was provided, but no " +
"matching files were found for them."))
return SpatialFiles(dirname, found, archive=archive)
def scan_file(file_name, scan_hint=None, charset=None):
'''get a list of SpatialFiles for the provided file'''
if not os.path.exists(file_name):
raise Exception(_("Could not access to uploaded data."))
dirname = os.path.dirname(file_name)
if zipfile.is_zipfile(file_name):
paths, kept_zip = _process_zip(file_name,
dirname,
scan_hint=scan_hint,
charset=charset)
archive = file_name if kept_zip else None
else:
paths = []
for p in os.listdir(dirname):
_f = os.path.join(dirname, p)
fixup_shp_columnnames(_f, charset)
paths.append(_f)
archive = None
if paths is not None:
safe_paths = _rename_files(paths)
else:
safe_paths = []
found = []
for file_type in types:
for path in safe_paths:
path_extension = os.path.splitext(path)[-1][1:]
hint_ok = (scan_hint is None or file_type.code == scan_hint or
scan_hint in file_type.aliases)
if file_type.matches(path_extension) and hint_ok:
_f = file_type.build_spatial_file(path, safe_paths)
found_paths = [f.base_file for f in found]
if path not in found_paths:
found.append(_f)
# detect xmls and assign if a single upload is found
xml_files = _find_file_type(safe_paths, extension='.xml')
if xml_files:
if len(found) == 1:
found[0].xml_files = xml_files
else:
raise Exception(_("One or more XML files was provided, but no " +
"matching files were found for them."))
# detect slds and assign if a single upload is found
sld_files = _find_file_type(safe_paths, extension='.sld')
if sld_files:
if len(found) == 1:
found[0].sld_files = sld_files
else:
raise Exception(_("One or more SLD files was provided, but no " +
"matching files were found for them."))
return SpatialFiles(dirname, found, archive=archive)
def test_fixup_shp_columnnames(self):
project_root = os.path.abspath(os.path.dirname(__file__))
layer_zip = os.path.join(project_root, "data", "ming_female_1.zip")
self.failUnless(zipfile.is_zipfile(layer_zip))
layer_shp = unzip_file(layer_zip)
expected_fieldnames = [
"ID", "_f", "__1", "__2", "m", "_", "_M2", "_M2_1", "l", "x", "y", "_WU", "_1",
]
_, _, fieldnames = fixup_shp_columnnames(layer_shp, "windows-1258")
inDriver = ogr.GetDriverByName("ESRI Shapefile")
inDataSource = inDriver.Open(layer_shp, 0)
inLayer = inDataSource.GetLayer()
inLayerDefn = inLayer.GetLayerDefn()
self.assertEqual(inLayerDefn.GetFieldCount(), len(expected_fieldnames))
for i, fn in enumerate(expected_fieldnames):
self.assertEqual(inLayerDefn.GetFieldDefn(i).GetName(), fn)
inDataSource.Destroy()
# Cleanup temp dir
shp_parent = os.path.dirname(layer_shp)
if shp_parent.startswith(tempfile.gettempdir()):
shutil.rmtree(shp_parent)
def save_step_view(req, session):
if req.method == 'GET':
return render(
req, 'upload/layer_upload.html', {
'async_upload': _ASYNC_UPLOAD,
'incomplete': Upload.objects.get_incomplete_uploads(req.user),
'charsets': CHARSETS
})
form = LayerUploadForm(req.POST, req.FILES)
overwrite = req.path_info.endswith('/replace')
target_store = None
if form.is_valid():
logger.debug(
f"valid_extensions: {form.cleaned_data['valid_extensions']}")
data_retriever = form.cleaned_data["data_retriever"]
relevant_files = _select_relevant_files(
form.cleaned_data["valid_extensions"],
data_retriever.get_paths(allow_transfer=False).values())
logger.debug(f"relevant_files: {relevant_files}")
base_file = data_retriever.get("base_file").get_path(
allow_transfer=False)
name, ext = os.path.splitext(os.path.basename(base_file))
logger.debug(f'Name: {name}, ext: {ext}')
logger.debug(f"base_file: {base_file}")
scan_hint = get_scan_hint(form.cleaned_data["valid_extensions"])
spatial_files = scan_file(base_file,
scan_hint=scan_hint,
charset=form.cleaned_data["charset"])
logger.debug(f"spatial_files: {spatial_files}")
if overwrite:
_layer = Layer.objects.filter(id=req.GET['layer_id'])
if _layer.exists():
name = _layer.first().name
target_store = _layer.first().store
import_session, upload = save_step(
req.user,
name,
spatial_files,
overwrite=overwrite,
store_spatial_files=form.cleaned_data.get('store_spatial_files',
True),
mosaic=form.cleaned_data['mosaic'] or scan_hint == 'zip-mosaic',
append_to_mosaic_opts=form.cleaned_data['append_to_mosaic_opts'],
append_to_mosaic_name=form.cleaned_data['append_to_mosaic_name'],
mosaic_time_regex=form.cleaned_data['mosaic_time_regex'],
mosaic_time_value=form.cleaned_data['mosaic_time_value'],
time_presentation=form.cleaned_data['time_presentation'],
time_presentation_res=form.cleaned_data['time_presentation_res'],
time_presentation_default_value=form.
cleaned_data['time_presentation_default_value'],
time_presentation_reference_value=form.
cleaned_data['time_presentation_reference_value'],
charset_encoding=form.cleaned_data["charset"],
target_store=target_store)
import_session.tasks[0].set_charset(form.cleaned_data["charset"])
sld = None
if spatial_files[0].sld_files:
sld = spatial_files[0].sld_files[0]
if not os.path.isfile(
os.path.join(data_retriever.temporary_folder,
spatial_files[0].base_file)):
tmp_files = [
f for f in os.listdir(data_retriever.temporary_folder)
if os.path.isfile(
os.path.join(data_retriever.temporary_folder, f))
]
for f in tmp_files:
if zipfile.is_zipfile(
os.path.join(data_retriever.temporary_folder, f)):
fixup_shp_columnnames(
os.path.join(data_retriever.temporary_folder, f),
form.cleaned_data["charset"],
tempdir=data_retriever.temporary_folder)
_log(f'provided sld is {sld}')
# upload_type = get_upload_type(base_file)
upload_session = UploaderSession(
tempdir=data_retriever.temporary_folder,
base_file=spatial_files,
name=upload.name,
charset=form.cleaned_data["charset"],
import_session=import_session,
layer_abstract=form.cleaned_data["abstract"],
layer_title=form.cleaned_data["layer_title"],
permissions=form.cleaned_data["permissions"],
import_sld_file=sld,
spatial_files_uploaded=form.cleaned_data['uploaded'],
upload_type=spatial_files[0].file_type.code,
time=form.cleaned_data['time'],
mosaic=form.cleaned_data['mosaic'],
append_to_mosaic_opts=form.cleaned_data['append_to_mosaic_opts'],
append_to_mosaic_name=form.cleaned_data['append_to_mosaic_name'],
mosaic_time_regex=form.cleaned_data['mosaic_time_regex'],
mosaic_time_value=form.cleaned_data['mosaic_time_value'],
user=upload.user)
Upload.objects.update_from_session(upload_session)
return next_step_response(req, upload_session, force_ajax=True)
else:
if hasattr(form, "data_retriever"):
form.data_retriever.delete_files()
errors = []
for e in form.errors.values():
errors.extend([escape(v) for v in e])
return error_response(req, errors=errors)
def save_step_view(req, session):
if req.method == 'GET':
return render(
req,
'upload/layer_upload.html',
{
'async_upload': _ASYNC_UPLOAD,
'incomplete': Upload.objects.get_incomplete_uploads(req.user),
'charsets': CHARSETS
}
)
form = LayerUploadForm(req.POST, req.FILES)
if form.is_valid():
tempdir = tempfile.mkdtemp(dir=settings.FILE_UPLOAD_TEMP_DIR)
logger.debug("valid_extensions: {}".format(form.cleaned_data["valid_extensions"]))
relevant_files = _select_relevant_files(
form.cleaned_data["valid_extensions"],
iter(req.FILES.values())
)
logger.debug("relevant_files: {}".format(relevant_files))
_write_uploaded_files_to_disk(tempdir, relevant_files)
base_file = os.path.join(tempdir, form.cleaned_data["base_file"].name)
name, ext = os.path.splitext(os.path.basename(base_file))
logger.debug('Name: {0}, ext: {1}'.format(name, ext))
logger.debug("base_file: {}".format(base_file))
scan_hint = get_scan_hint(form.cleaned_data["valid_extensions"])
spatial_files = scan_file(
base_file,
scan_hint=scan_hint,
charset=form.cleaned_data["charset"]
)
logger.debug("spatial_files: {}".format(spatial_files))
import_session = save_step(
req.user,
name,
spatial_files,
overwrite=False,
mosaic=form.cleaned_data['mosaic'] or scan_hint == 'zip-mosaic',
append_to_mosaic_opts=form.cleaned_data['append_to_mosaic_opts'],
append_to_mosaic_name=form.cleaned_data['append_to_mosaic_name'],
mosaic_time_regex=form.cleaned_data['mosaic_time_regex'],
mosaic_time_value=form.cleaned_data['mosaic_time_value'],
time_presentation=form.cleaned_data['time_presentation'],
time_presentation_res=form.cleaned_data['time_presentation_res'],
time_presentation_default_value=form.cleaned_data['time_presentation_default_value'],
time_presentation_reference_value=form.cleaned_data['time_presentation_reference_value'],
charset_encoding=form.cleaned_data["charset"]
)
import_session.tasks[0].set_charset(form.cleaned_data["charset"])
sld = None
if spatial_files[0].sld_files:
sld = spatial_files[0].sld_files[0]
if not os.path.isfile(os.path.join(tempdir, spatial_files[0].base_file)):
tmp_files = [f for f in os.listdir(tempdir) if os.path.isfile(os.path.join(tempdir, f))]
for f in tmp_files:
if zipfile.is_zipfile(os.path.join(tempdir, f)):
fixup_shp_columnnames(os.path.join(tempdir, f),
form.cleaned_data["charset"],
tempdir=tempdir)
_log('provided sld is %s' % sld)
# upload_type = get_upload_type(base_file)
upload_session = UploaderSession(
tempdir=tempdir,
base_file=spatial_files,
name=name,
charset=form.cleaned_data["charset"],
import_session=import_session,
layer_abstract=form.cleaned_data["abstract"],
layer_title=form.cleaned_data["layer_title"],
permissions=form.cleaned_data["permissions"],
import_sld_file=sld,
upload_type=spatial_files[0].file_type.code,
time=form.cleaned_data['time'],
mosaic=form.cleaned_data['mosaic'],
append_to_mosaic_opts=form.cleaned_data['append_to_mosaic_opts'],
append_to_mosaic_name=form.cleaned_data['append_to_mosaic_name'],
mosaic_time_regex=form.cleaned_data['mosaic_time_regex'],
mosaic_time_value=form.cleaned_data['mosaic_time_value'],
user=req.user
)
Upload.objects.update_from_session(upload_session)
return next_step_response(req, upload_session, force_ajax=True)
else:
errors = []
for e in form.errors.values():
errors.extend([escape(v) for v in e])
return error_response(req, errors=errors)
def save_step_view(req, session):
if req.method == 'GET':
return render(
req,
'upload/layer_upload.html',
{
'async_upload': _ASYNC_UPLOAD,
'incomplete': Upload.objects.get_incomplete_uploads(req.user),
'charsets': CHARSETS
}
)
form = LayerUploadForm(req.POST, req.FILES)
if form.is_valid():
tempdir = tempfile.mkdtemp(dir=settings.FILE_UPLOAD_TEMP_DIR)
logger.info("valid_extensions: {}".format(form.cleaned_data["valid_extensions"]))
relevant_files = _select_relevant_files(
form.cleaned_data["valid_extensions"],
req.FILES.itervalues()
)
logger.info("relevant_files: {}".format(relevant_files))
_write_uploaded_files_to_disk(tempdir, relevant_files)
base_file = os.path.join(tempdir, form.cleaned_data["base_file"].name)
name, ext = os.path.splitext(os.path.basename(base_file))
logger.debug('Name: {0}, ext: {1}'.format(name, ext))
logger.debug("base_file: {}".format(base_file))
scan_hint = get_scan_hint(form.cleaned_data["valid_extensions"])
spatial_files = scan_file(
base_file,
scan_hint=scan_hint,
charset=form.cleaned_data["charset"]
)
logger.info("spatial_files: {}".format(spatial_files))
import_session = save_step(
req.user,
name,
spatial_files,
overwrite=False,
mosaic=form.cleaned_data['mosaic'] or scan_hint == 'zip-mosaic',
append_to_mosaic_opts=form.cleaned_data['append_to_mosaic_opts'],
append_to_mosaic_name=form.cleaned_data['append_to_mosaic_name'],
mosaic_time_regex=form.cleaned_data['mosaic_time_regex'],
mosaic_time_value=form.cleaned_data['mosaic_time_value'],
time_presentation=form.cleaned_data['time_presentation'],
time_presentation_res=form.cleaned_data['time_presentation_res'],
time_presentation_default_value=form.cleaned_data['time_presentation_default_value'],
time_presentation_reference_value=form.cleaned_data['time_presentation_reference_value']
)
sld = None
if spatial_files[0].sld_files:
sld = spatial_files[0].sld_files[0]
if not os.path.isfile(os.path.join(tempdir, spatial_files[0].base_file)):
tmp_files = [f for f in os.listdir(tempdir) if os.path.isfile(os.path.join(tempdir, f))]
for f in tmp_files:
if zipfile.is_zipfile(os.path.join(tempdir, f)):
fixup_shp_columnnames(os.path.join(tempdir, f),
form.cleaned_data["charset"],
tempdir=tempdir)
_log('provided sld is %s' % sld)
# upload_type = get_upload_type(base_file)
upload_session = UploaderSession(
tempdir=tempdir,
base_file=spatial_files,
name=name,
import_session=import_session,
layer_abstract=form.cleaned_data["abstract"],
layer_title=form.cleaned_data["layer_title"],
permissions=form.cleaned_data["permissions"],
import_sld_file=sld,
upload_type=spatial_files[0].file_type.code,
time=form.cleaned_data['time'],
mosaic=form.cleaned_data['mosaic'],
append_to_mosaic_opts=form.cleaned_data['append_to_mosaic_opts'],
append_to_mosaic_name=form.cleaned_data['append_to_mosaic_name'],
mosaic_time_regex=form.cleaned_data['mosaic_time_regex'],
mosaic_time_value=form.cleaned_data['mosaic_time_value'],
user=req.user
)
req.session[str(upload_session.import_session.id)] = upload_session
_log('saved session : %s',
req.session[str(upload_session.import_session.id)])
return next_step_response(req, upload_session, force_ajax=True)
else:
errors = []
for e in form.errors.values():
errors.extend([escape(v) for v in e])
return error_response(req, errors=errors)