def test_get_meta(self, open_dataset_mock, isfile):
dataset_path = "/path/to/dataset"
isfile.return_value = True
self.task_process.return_value = Mock(exitcode=0)
mock_open_dataset = Mock(spec=gdal.Dataset)
mock_open_dataset.RasterCount = 0
open_dataset_mock.return_value = mock_open_dataset
mock_open_dataset.GetDriver.return_value.ShortName = "gtiff"
expected_meta = {"driver": "gtiff", "is_raster": True, "nodata": None}
returned_meta = get_meta(dataset_path)
self.assertEqual(expected_meta, returned_meta)
mock_open_dataset.RasterCount = 2
mock_open_dataset.GetRasterBand.return_value.GetNoDataValue.return_value = -32768.0
expected_meta = {
"driver": "gtiff",
"is_raster": True,
"nodata": -32768.0
}
returned_meta = get_meta(dataset_path)
self.assertEqual(expected_meta, returned_meta)
mock_open_dataset = Mock(spec=ogr.DataSource)
open_dataset_mock.return_value = mock_open_dataset
mock_open_dataset.GetDriver.return_value.GetName.return_value = "gpkg"
expected_meta = {"driver": "gpkg", "is_raster": False, "nodata": None}
returned_meta = get_meta(dataset_path)
self.assertEqual(expected_meta, returned_meta)
open_dataset_mock.return_value = None
expected_meta = {"driver": None, "is_raster": None, "nodata": None}
returned_meta = get_meta(dataset_path)
self.assertEqual(expected_meta, returned_meta)
def test_get_meta(self, open_ds_mock, isfile):
dataset_path = "/path/to/dataset"
isfile.return_value = True
self.task_process.return_value = Mock(exitcode=0)
ds_mock = Mock(spec=gdal.Dataset)
ds_mock.RasterCount = 0
open_ds_mock.return_value = ds_mock
ds_mock.GetDriver.return_value.ShortName = 'gtiff'
expected_meta = {'driver': 'gtiff', 'is_raster': True, 'nodata': None}
returned_meta = get_meta(dataset_path)
self.assertEqual(expected_meta, returned_meta)
ds_mock.RasterCount = 2
ds_mock.GetRasterBand.return_value.GetNoDataValue.return_value = -32768.0
expected_meta = {
'driver': 'gtiff',
'is_raster': True,
'nodata': -32768.0
}
returned_meta = get_meta(dataset_path)
self.assertEqual(expected_meta, returned_meta)
ds_mock = Mock(spec=ogr.DataSource)
open_ds_mock.return_value = ds_mock
ds_mock.GetDriver.return_value.GetName.return_value = 'gpkg'
expected_meta = {'driver': 'gpkg', 'is_raster': False, 'nodata': None}
returned_meta = get_meta(dataset_path)
self.assertEqual(expected_meta, returned_meta)
open_ds_mock.return_value = None
expected_meta = {'driver': None, 'is_raster': None, 'nodata': None}
returned_meta = get_meta(dataset_path)
self.assertEqual(expected_meta, returned_meta)
def test_get_meta(self, open_ds_mock, isfile):
dataset_path = "/path/to/dataset"
isfile.return_value = True
self.task_process.return_value = Mock(exitcode=0)
ds_mock = Mock(spec=gdal.Dataset)
ds_mock.RasterCount = 0
open_ds_mock.return_value = ds_mock
ds_mock.GetDriver.return_value.ShortName = 'gtiff'
expected_meta = {'driver': 'gtiff', 'is_raster': True, 'nodata': None}
returned_meta = get_meta(dataset_path)
self.assertEqual(expected_meta, returned_meta)
ds_mock.RasterCount = 2
ds_mock.GetRasterBand.return_value.GetNoDataValue.return_value = -32768.0
expected_meta = {'driver': 'gtiff', 'is_raster': True, 'nodata': -32768.0}
returned_meta = get_meta(dataset_path)
self.assertEqual(expected_meta, returned_meta)
ds_mock = Mock(spec=ogr.DataSource)
open_ds_mock.return_value = ds_mock
ds_mock.GetDriver.return_value.GetName.return_value = 'gpkg'
expected_meta = {'driver': 'gpkg', 'is_raster': False, 'nodata': None}
returned_meta = get_meta(dataset_path)
self.assertEqual(expected_meta, returned_meta)
open_ds_mock.return_value = None
expected_meta = {'driver': None, 'is_raster': None, 'nodata': None}
returned_meta = get_meta(dataset_path)
self.assertEqual(expected_meta, returned_meta)
def file_to_geojson(in_path: str):
"""
:param in_path: A str path to a file.
:return: A geojson object if available
"""
dir_name = os.path.dirname(in_path)
file_path = pathlib.Path(in_path)
try:
out_path = os.path.join(dir_name,
"out_{0}.geojson".format(file_path.stem))
if file_path.suffix == ".zip":
if unzip_file(in_path, dir_name):
has_shp = False
for unzipped_file in os.listdir(dir_name):
if unzipped_file.endswith(".shp"):
in_path = os.path.join(dir_name, unzipped_file)
has_shp = True
break
if not has_shp:
raise Exception("Zip file does not contain a shp")
meta = get_meta(in_path, is_raster=False)
if not meta["driver"] or meta["is_raster"]:
out_path = polygonize(in_path, out_path)
else:
out_path = convert_vector(in_path, out_path, driver="geojson")
if os.path.exists(out_path):
geojson = read_json_file(out_path)
return geojson
raise Exception("An unknown error occurred while processing the file")
except Exception as e:
logger.error(e, exc_info=True)
raise e
finally:
if os.path.exists(dir_name):
shutil.rmtree(dir_name)
def file_to_geojson(in_memory_file):
"""
:param in_memory_file: A WSGI In memory file
:return: A geojson object if available
"""
stage_dir = settings.EXPORT_STAGING_ROOT.rstrip('\/')
uid = str(uuid4())
dir = os.path.join(stage_dir, uid)
try:
os.mkdir(dir)
file_name = in_memory_file.name
file_name, file_extension = os.path.splitext(file_name)
if not file_name or not file_extension:
raise Exception('No file type detected')
# Remove all non-word characters
file_name = re.sub(r"[^\w\s]", '', file_name)
# Replace all whitespace with a single underscore
file_name = re.sub(r"\s+", '_', file_name).lower()
in_path = os.path.join(dir, 'in_{0}{1}'.format(file_name, file_extension))
out_path = os.path.join(dir, 'out_{0}.geojson'.format(file_name))
write_uploaded_file(in_memory_file, in_path)
if file_extension == '.zip':
if unzip_file(in_path, dir):
has_shp = False
for unzipped_file in os.listdir(dir):
if unzipped_file.endswith('.shp'):
in_path = os.path.join(dir, unzipped_file)
has_shp = True
break
if not has_shp:
raise Exception('Zip file does not contain a shp')
meta = get_meta(in_path)
if not meta['driver'] or meta['is_raster']:
raise Exception("Could not find the proper driver to handle this file")
cmd_template = Template("ogr2ogr -f $fmt $out_ds $in_ds")
cmd = cmd_template.safe_substitute({
'fmt': 'geojson',
'out_ds': out_path,
'in_ds': in_path
})
try:
proc = subprocess.Popen(cmd, shell=True, executable='/bin/bash')
proc.wait()
except Exception as e:
logger.debug(e)
raise Exception('Failed to convert file')
if os.path.exists(out_path):
geojson = read_json_file(out_path)
return geojson
raise Exception('An unknown error occurred while processing the file')
except Exception as e:
logger.error(e)
raise e
finally:
if os.path.exists(dir):
shutil.rmtree(dir)
def get_coverage_with_requests(self):
logger.info("Using admin configuration for the WCS request.")
service = self.config.get("service")
params = self.config.get("params")
if not service:
raise Exception(
"A service key needs to be defined to include the scale of source in meters"
)
coverages = service.get("coverages", params.get("COVERAGE"))
coverages = str(coverages).split(",")
if not coverages:
logger.error(
"No coverages were specified for this provider, "
"please specify `coverages` under service or `COVERAGE` under params."
# NOQA
)
raise Exception("Data source incorrectly configured.")
scale = float(service.get("scale"))
params["service"] = "WCS"
width, height = gdalutils.get_dimensions(self.bbox, scale)
tile_bboxes = gdalutils.get_chunked_bbox(self.bbox, (width, height))
geotiffs = []
session = get_or_update_session(slug=self.slug, **self.config)
for idx, coverage in enumerate(coverages):
params["COVERAGE"] = coverage
file_path, ext = os.path.splitext(self.out)
try:
for (
_bbox_idx,
_tile_bbox,
) in enumerate(tile_bboxes):
outfile = "{0}-{1}-{2}{3}".format(file_path, idx,
_bbox_idx, ext)
try:
os.remove(outfile)
except OSError:
pass
# Setting this to arbitrarily high values improves the computed
# resolution but makes the requests slow down.
# If it is set in the config, use that value, otherwise compute approximate res based on scale
if self.config.get("tile_size", None) is None:
tile_x, tile_y = gdalutils.get_dimensions(
_tile_bbox, scale)
params["width"] = tile_x
params["height"] = tile_y
else:
params["width"] = self.config.get("tile_size")
params["height"] = self.config.get("tile_size")
params["bbox"] = ",".join(map(str, _tile_bbox))
req = session.get(self.service_url,
params=params,
stream=True)
try:
size = int(req.headers.get("content-length"))
except (ValueError, TypeError):
if req.content:
size = len(req.content)
else:
raise Exception(
"Overpass Query failed to return any data")
if not req:
logger.error(req.content)
raise Exception("WCS request for {0} failed.".format(
self.name))
CHUNK = 1024 * 1024 * 2 # 2MB chunks
from audit_logging.file_logging import logging_open
with logging_open(outfile,
"wb",
user_details=self.user_details) as fd:
for chunk in req.iter_content(CHUNK):
fd.write(chunk)
size += CHUNK
geotiffs += [outfile]
except Exception as e:
logger.error(e)
raise Exception("There was an error writing the file to disk.")
if len(geotiffs) > 1:
self.out = gdalutils.merge_geotiffs(geotiffs,
self.out,
task_uid=self.task_uid)
else:
shutil.copy(geotiffs[0], self.out)
if not os.path.isfile(self.out):
raise Exception("Nothing was returned from the WCS service.")
if not gdalutils.get_meta(self.out).get("is_raster"):
with open(self.out, "r") as output_file:
logger.error("Content of failed WCS request")
logger.error(output_file.read())
raise Exception("The service failed to return a proper response")