def test_merge_geotiffs(self):
in_files = ["1.tif", "2.tif", "3.tif", "4.tif"]
out_file = "merged.tif"
task_uid = "1234"
result = merge_geotiffs(in_files, out_file, task_uid=task_uid)
self.task_process.start_process.called_once()
self.assertEqual(out_file, result)
with self.assertRaises(Exception):
self.task_process().start_process.side_effect = Exception("Error")
merge_geotiffs(in_files, out_file, task_uid=task_uid)
def test_merge_geotiffs(self, mock_taskprocess):
in_files = ['1.tif', '2.tif', '3.tif', '4.tif']
out_file = 'merged.tif'
task_uid = "1234"
expected_command = "gdalwarp {0} {1}".format(' '.join(in_files), out_file)
mock_tp = MagicMock()
mock_tp.exitcode = 0
mock_taskprocess.return_value = mock_tp
result = merge_geotiffs(in_files, out_file, task_uid=task_uid)
mock_tp.start_process.called_once_with(expected_command, shell=True, executable="/bin/bash",
stdout=ANY, stderr=ANY)
self.assertEqual(out_file, result)
with self.assertRaises(Exception):
mock_tp.exitcode = 1
merge_geotiffs(in_files, out_file, task_uid=task_uid)
def get_coverage_with_requests(self):
logger.info("Using admin configuration for the WCS request.")
service = self.config.get('service')
params = self.config.get('params')
if not service:
raise Exception('A service key needs to be defined to include the scale of source in meters')
coverages = service.get('coverages', params.get('COVERAGE'))
coverages = str(coverages).split(',')
if not coverages:
logger.error('No coverages were specified for this provider, please specify `coverages` under service or `COVERAGE` under params.')
raise Exception("Data source incorrectly configured.")
logger.info("Getting Dimensions...")
width, height = get_dimensions(self.bbox, float(service.get('scale')))
logger.info("{}, {}".format(width, height))
params['width'] = str(width)
params['height'] = str(height)
params['service'] = 'WCS'
params['bbox'] = ','.join(map(str, self.bbox))
geotiffs = []
for idx, coverage in enumerate(coverages):
params['COVERAGE'] = coverage
file_path, ext = os.path.splitext(self.out)
outfile = '{0}-{1}{2}'.format(file_path, idx, ext)
try:
os.remove(outfile)
except OSError:
pass
try:
req = auth_requests.get(self.service_url, params=params, slug=self.slug, stream=True,
verify=getattr(settings, 'SSL_VERIFICATION', True))
logger.info("Getting the coverage: {0}".format(req.url))
try:
size = int(req.headers.get('content-length'))
except (ValueError, TypeError):
if req.content:
size = len(req.content)
else:
raise Exception("Overpass Query failed to return any data")
if not req:
logger.error(req.content)
raise Exception("WCS request for {0} failed.".format(self.name))
CHUNK = 1024 * 1024 * 2 # 2MB chunks
from audit_logging.file_logging import logging_open
with logging_open(outfile, 'wb', user_details=self.user_details) as fd:
for chunk in req.iter_content(CHUNK):
fd.write(chunk)
size += CHUNK
geotiffs += [outfile]
except Exception as e:
logger.error(e)
raise Exception("There was an error writing the file to disk.")
if len(geotiffs) > 1:
self.out = merge_geotiffs(geotiffs, self.out, task_uid=self.task_uid)
else:
shutil.copy(geotiffs[0], self.out)
if not os.path.isfile(self.out):
raise Exception("Nothing was returned from the WCS service.")
def test_merge_geotiffs(self, mock_taskprocess):
in_files = ['1.tif', '2.tif', '3.tif', '4.tif']
out_file = 'merged.tif'
task_uid = "1234"
expected_command = "gdalwarp {0} {1}".format(' '.join(in_files),
out_file)
mock_tp = MagicMock()
mock_tp.exitcode = 0
mock_taskprocess.return_value = mock_tp
result = merge_geotiffs(in_files, out_file, task_uid=task_uid)
mock_tp.start_process.called_once_with(expected_command,
shell=True,
executable="/bin/bash",
stdout=ANY,
stderr=ANY)
self.assertEqual(out_file, result)
with self.assertRaises(Exception):
mock_tp.exitcode = 1
merge_geotiffs(in_files, out_file, task_uid=task_uid)
def get_coverage_with_requests(self):
logger.info("Using admin configuration for the WCS request.")
service = self.config.get('service')
params = self.config.get('params')
if not service:
raise Exception(
'A service key needs to be defined to include the scale of source in meters'
)
coverages = service.get('coverages', params.get('COVERAGE'))
coverages = str(coverages).split(',')
if not coverages:
logger.error(
'No coverages were specified for this provider, please specify `coverages` under service or `COVERAGE` under params.'
)
raise Exception("Data source incorrectly configured.")
logger.info("Getting Dimensions...")
width, height = get_dimensions(self.bbox, float(service.get('scale')))
logger.info("{}, {}".format(width, height))
params['width'] = str(width)
params['height'] = str(height)
params['service'] = 'WCS'
params['bbox'] = ','.join(map(str, self.bbox))
geotiffs = []
for idx, coverage in enumerate(coverages):
params['COVERAGE'] = coverage
file_path, ext = os.path.splitext(self.out)
outfile = '{0}-{1}{2}'.format(file_path, idx, ext)
try:
os.remove(outfile)
except OSError:
pass
try:
req = auth_requests.get(self.service_url,
params=params,
slug=self.slug,
stream=True,
verify=getattr(settings,
'SSL_VERIFICATION',
True))
logger.info("Getting the coverage: {0}".format(req.url))
try:
size = int(req.headers.get('content-length'))
except (ValueError, TypeError):
if req.content:
size = len(req.content)
else:
raise Exception(
"Overpass Query failed to return any data")
if not req:
logger.error(req.content)
raise Exception("WCS request for {0} failed.".format(
self.name))
CHUNK = 1024 * 1024 * 2 # 2MB chunks
from audit_logging.file_logging import logging_open
with logging_open(outfile,
'wb',
user_details=self.user_details) as fd:
for chunk in req.iter_content(CHUNK):
fd.write(chunk)
size += CHUNK
geotiffs += [outfile]
except Exception as e:
logger.error(e)
raise Exception("There was an error writing the file to disk.")
if len(geotiffs) > 1:
self.out = merge_geotiffs(geotiffs,
self.out,
task_uid=self.task_uid)
else:
shutil.copy(geotiffs[0], self.out)
if not os.path.isfile(self.out):
raise Exception("Nothing was returned from the WCS service.")
def get_coverage_with_requests(self):
logger.info("Using admin configuration for the WCS request.")
service = self.config.get("service")
params = self.config.get("params")
if not service:
raise Exception(
"A service key needs to be defined to include the scale of source in meters"
)
coverages = service.get("coverages", params.get("COVERAGE"))
coverages = str(coverages).split(",")
if not coverages:
logger.error(
"No coverages were specified for this provider, "
"please specify `coverages` under service or `COVERAGE` under params."
# NOQA
)
raise Exception("Data source incorrectly configured.")
scale = float(service.get("scale"))
params["service"] = "WCS"
width, height = gdalutils.get_dimensions(self.bbox, scale)
tile_bboxes = gdalutils.get_chunked_bbox(self.bbox, (width, height))
geotiffs = []
session = get_or_update_session(slug=self.slug, **self.config)
for idx, coverage in enumerate(coverages):
params["COVERAGE"] = coverage
file_path, ext = os.path.splitext(self.out)
try:
for (
_bbox_idx,
_tile_bbox,
) in enumerate(tile_bboxes):
outfile = "{0}-{1}-{2}{3}".format(file_path, idx,
_bbox_idx, ext)
try:
os.remove(outfile)
except OSError:
pass
# Setting this to arbitrarily high values improves the computed
# resolution but makes the requests slow down.
# If it is set in the config, use that value, otherwise compute approximate res based on scale
if self.config.get("tile_size", None) is None:
tile_x, tile_y = gdalutils.get_dimensions(
_tile_bbox, scale)
params["width"] = tile_x
params["height"] = tile_y
else:
params["width"] = self.config.get("tile_size")
params["height"] = self.config.get("tile_size")
params["bbox"] = ",".join(map(str, _tile_bbox))
req = session.get(self.service_url,
params=params,
stream=True)
try:
size = int(req.headers.get("content-length"))
except (ValueError, TypeError):
if req.content:
size = len(req.content)
else:
raise Exception(
"Overpass Query failed to return any data")
if not req:
logger.error(req.content)
raise Exception("WCS request for {0} failed.".format(
self.name))
CHUNK = 1024 * 1024 * 2 # 2MB chunks
from audit_logging.file_logging import logging_open
with logging_open(outfile,
"wb",
user_details=self.user_details) as fd:
for chunk in req.iter_content(CHUNK):
fd.write(chunk)
size += CHUNK
geotiffs += [outfile]
except Exception as e:
logger.error(e)
raise Exception("There was an error writing the file to disk.")
if len(geotiffs) > 1:
self.out = gdalutils.merge_geotiffs(geotiffs,
self.out,
task_uid=self.task_uid)
else:
shutil.copy(geotiffs[0], self.out)
if not os.path.isfile(self.out):
raise Exception("Nothing was returned from the WCS service.")
if not gdalutils.get_meta(self.out).get("is_raster"):
with open(self.out, "r") as output_file:
logger.error("Content of failed WCS request")
logger.error(output_file.read())
raise Exception("The service failed to return a proper response")