def test_merge_geotiffs(self): in_files = ["1.tif", "2.tif", "3.tif", "4.tif"] out_file = "merged.tif" task_uid = "1234" result = merge_geotiffs(in_files, out_file, task_uid=task_uid) self.task_process.start_process.called_once() self.assertEqual(out_file, result) with self.assertRaises(Exception): self.task_process().start_process.side_effect = Exception("Error") merge_geotiffs(in_files, out_file, task_uid=task_uid)
def test_merge_geotiffs(self, mock_taskprocess): in_files = ['1.tif', '2.tif', '3.tif', '4.tif'] out_file = 'merged.tif' task_uid = "1234" expected_command = "gdalwarp {0} {1}".format(' '.join(in_files), out_file) mock_tp = MagicMock() mock_tp.exitcode = 0 mock_taskprocess.return_value = mock_tp result = merge_geotiffs(in_files, out_file, task_uid=task_uid) mock_tp.start_process.called_once_with(expected_command, shell=True, executable="/bin/bash", stdout=ANY, stderr=ANY) self.assertEqual(out_file, result) with self.assertRaises(Exception): mock_tp.exitcode = 1 merge_geotiffs(in_files, out_file, task_uid=task_uid)
def get_coverage_with_requests(self): logger.info("Using admin configuration for the WCS request.") service = self.config.get('service') params = self.config.get('params') if not service: raise Exception('A service key needs to be defined to include the scale of source in meters') coverages = service.get('coverages', params.get('COVERAGE')) coverages = str(coverages).split(',') if not coverages: logger.error('No coverages were specified for this provider, please specify `coverages` under service or `COVERAGE` under params.') raise Exception("Data source incorrectly configured.") logger.info("Getting Dimensions...") width, height = get_dimensions(self.bbox, float(service.get('scale'))) logger.info("{}, {}".format(width, height)) params['width'] = str(width) params['height'] = str(height) params['service'] = 'WCS' params['bbox'] = ','.join(map(str, self.bbox)) geotiffs = [] for idx, coverage in enumerate(coverages): params['COVERAGE'] = coverage file_path, ext = os.path.splitext(self.out) outfile = '{0}-{1}{2}'.format(file_path, idx, ext) try: os.remove(outfile) except OSError: pass try: req = auth_requests.get(self.service_url, params=params, slug=self.slug, stream=True, verify=getattr(settings, 'SSL_VERIFICATION', True)) logger.info("Getting the coverage: {0}".format(req.url)) try: size = int(req.headers.get('content-length')) except (ValueError, TypeError): if req.content: size = len(req.content) else: raise Exception("Overpass Query failed to return any data") if not req: logger.error(req.content) raise Exception("WCS request for {0} failed.".format(self.name)) CHUNK = 1024 * 1024 * 2 # 2MB chunks from audit_logging.file_logging import logging_open with logging_open(outfile, 'wb', user_details=self.user_details) as fd: for chunk in req.iter_content(CHUNK): fd.write(chunk) size += CHUNK geotiffs += [outfile] except Exception as e: logger.error(e) raise Exception("There was an error writing the file to disk.") if len(geotiffs) > 1: self.out = merge_geotiffs(geotiffs, self.out, task_uid=self.task_uid) else: shutil.copy(geotiffs[0], self.out) if not os.path.isfile(self.out): raise Exception("Nothing was returned from the WCS service.")
def get_coverage_with_requests(self): logger.info("Using admin configuration for the WCS request.") service = self.config.get('service') params = self.config.get('params') if not service: raise Exception( 'A service key needs to be defined to include the scale of source in meters' ) coverages = service.get('coverages', params.get('COVERAGE')) coverages = str(coverages).split(',') if not coverages: logger.error( 'No coverages were specified for this provider, please specify `coverages` under service or `COVERAGE` under params.' ) raise Exception("Data source incorrectly configured.") logger.info("Getting Dimensions...") width, height = get_dimensions(self.bbox, float(service.get('scale'))) logger.info("{}, {}".format(width, height)) params['width'] = str(width) params['height'] = str(height) params['service'] = 'WCS' params['bbox'] = ','.join(map(str, self.bbox)) geotiffs = [] for idx, coverage in enumerate(coverages): params['COVERAGE'] = coverage file_path, ext = os.path.splitext(self.out) outfile = '{0}-{1}{2}'.format(file_path, idx, ext) try: os.remove(outfile) except OSError: pass try: req = auth_requests.get(self.service_url, params=params, slug=self.slug, stream=True, verify=getattr(settings, 'SSL_VERIFICATION', True)) logger.info("Getting the coverage: {0}".format(req.url)) try: size = int(req.headers.get('content-length')) except (ValueError, TypeError): if req.content: size = len(req.content) else: raise Exception( "Overpass Query failed to return any data") if not req: logger.error(req.content) raise Exception("WCS request for {0} failed.".format( self.name)) CHUNK = 1024 * 1024 * 2 # 2MB chunks from audit_logging.file_logging import logging_open with logging_open(outfile, 'wb', user_details=self.user_details) as fd: for chunk in req.iter_content(CHUNK): fd.write(chunk) size += CHUNK geotiffs += [outfile] except Exception as e: logger.error(e) raise Exception("There was an error writing the file to disk.") if len(geotiffs) > 1: self.out = merge_geotiffs(geotiffs, self.out, task_uid=self.task_uid) else: shutil.copy(geotiffs[0], self.out) if not os.path.isfile(self.out): raise Exception("Nothing was returned from the WCS service.")
def get_coverage_with_requests(self): logger.info("Using admin configuration for the WCS request.") service = self.config.get("service") params = self.config.get("params") if not service: raise Exception( "A service key needs to be defined to include the scale of source in meters" ) coverages = service.get("coverages", params.get("COVERAGE")) coverages = str(coverages).split(",") if not coverages: logger.error( "No coverages were specified for this provider, " "please specify `coverages` under service or `COVERAGE` under params." # NOQA ) raise Exception("Data source incorrectly configured.") scale = float(service.get("scale")) params["service"] = "WCS" width, height = gdalutils.get_dimensions(self.bbox, scale) tile_bboxes = gdalutils.get_chunked_bbox(self.bbox, (width, height)) geotiffs = [] session = get_or_update_session(slug=self.slug, **self.config) for idx, coverage in enumerate(coverages): params["COVERAGE"] = coverage file_path, ext = os.path.splitext(self.out) try: for ( _bbox_idx, _tile_bbox, ) in enumerate(tile_bboxes): outfile = "{0}-{1}-{2}{3}".format(file_path, idx, _bbox_idx, ext) try: os.remove(outfile) except OSError: pass # Setting this to arbitrarily high values improves the computed # resolution but makes the requests slow down. # If it is set in the config, use that value, otherwise compute approximate res based on scale if self.config.get("tile_size", None) is None: tile_x, tile_y = gdalutils.get_dimensions( _tile_bbox, scale) params["width"] = tile_x params["height"] = tile_y else: params["width"] = self.config.get("tile_size") params["height"] = self.config.get("tile_size") params["bbox"] = ",".join(map(str, _tile_bbox)) req = session.get(self.service_url, params=params, stream=True) try: size = int(req.headers.get("content-length")) except (ValueError, TypeError): if req.content: size = len(req.content) else: raise Exception( "Overpass Query failed to return any data") if not req: logger.error(req.content) raise Exception("WCS request for {0} failed.".format( self.name)) CHUNK = 1024 * 1024 * 2 # 2MB chunks from audit_logging.file_logging import logging_open with logging_open(outfile, "wb", user_details=self.user_details) as fd: for chunk in req.iter_content(CHUNK): fd.write(chunk) size += CHUNK geotiffs += [outfile] except Exception as e: logger.error(e) raise Exception("There was an error writing the file to disk.") if len(geotiffs) > 1: self.out = gdalutils.merge_geotiffs(geotiffs, self.out, task_uid=self.task_uid) else: shutil.copy(geotiffs[0], self.out) if not os.path.isfile(self.out): raise Exception("Nothing was returned from the WCS service.") if not gdalutils.get_meta(self.out).get("is_raster"): with open(self.out, "r") as output_file: logger.error("Content of failed WCS request") logger.error(output_file.read()) raise Exception("The service failed to return a proper response")